diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml
index a801779549..e10578db2e 100644
--- a/.github/workflows/docker-push.yml
+++ b/.github/workflows/docker-push.yml
@@ -101,7 +101,7 @@ jobs:
- uses: actions/setup-node@v4
with:
- node-version: 22.18.0
+ node-version: 22.22.3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index ffc6c2187d..98d75c1990 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -22,7 +22,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- node-version: [22.18.0]
+ node-version: [22.22.3]
runtime:
- mode: v1
force-v2-all: ''
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index 1460568160..dd040178f0 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -18,7 +18,7 @@ jobs:
strategy:
matrix:
- node-version: [22.18.0]
+ node-version: [22.22.3]
steps:
- uses: actions/checkout@v4
diff --git a/.github/workflows/manual-preview.yml b/.github/workflows/manual-preview.yml
index d0e7d7adc4..5ff9de2a2d 100644
--- a/.github/workflows/manual-preview.yml
+++ b/.github/workflows/manual-preview.yml
@@ -66,7 +66,7 @@ jobs:
- uses: actions/setup-node@v4
with:
- node-version: 22.18.0
+ node-version: 22.22.3
- name: ⚙️ Install zx
run: npm install -g zx
diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml
index 4107d07797..8820405e3f 100644
--- a/.github/workflows/publish-release.yml
+++ b/.github/workflows/publish-release.yml
@@ -11,11 +11,6 @@ on:
description: 'Final English release note in Markdown'
required: true
type: string
- media_sources:
- description: 'JSON map from original media URL to downloadable source metadata'
- required: false
- default: '{}'
- type: string
publish_latest:
description: 'Whether GitHub should mark this release as latest'
required: false
@@ -36,12 +31,6 @@ jobs:
ref: develop
fetch-depth: 0
- - name: Install FFmpeg when videos are present
- if: contains(inputs.release_note, '.mp4') || contains(inputs.release_note, '.webm') || contains(inputs.release_note, '.mov') || contains(inputs.release_note, '.m4v') || contains(inputs.release_note, '.ogg')
- run: |
- sudo apt-get update
- sudo apt-get install -y ffmpeg
-
- name: Build release body
env:
RELEASE_NOTE: ${{ inputs.release_note }}
@@ -49,7 +38,6 @@ jobs:
printf '%s\n' "$RELEASE_NOTE" > release-body.md
- name: Create or update GitHub release
- id: release
env:
GH_TOKEN: ${{ github.token }}
RELEASE_ID: ${{ inputs.release_id }}
@@ -73,7 +61,7 @@ jobs:
}
upsert_release() {
- local release_body release_payload response_file release_api_id release_url
+ local release_body release_payload response_file release_api_id
response_file="$(mktemp)"
release_body="$(cat release-body.md)"
release_payload="$(jq -nc \
@@ -103,200 +91,13 @@ jobs:
--input - <<<"${release_payload}" >"${response_file}"
fi
- release_api_id="$(jq -r '.id' "${response_file}")"
release_url="$(jq -r '.html_url // empty' "${response_file}")"
- if [ -z "${release_url}" ] || [ -z "${release_api_id}" ]; then
- echo "Missing release id/html_url in GitHub release response" >&2
+ if [ -z "${release_url}" ]; then
+ echo "Missing html_url in GitHub release response" >&2
cat "${response_file}" >&2
return 1
fi
-
- echo "release_api_id=${release_api_id}" >> "${GITHUB_OUTPUT}"
- echo "release_url=${release_url}" >> "${GITHUB_OUTPUT}"
}
ensure_tag
upsert_release
-
- - name: Render media links and upload GIF previews
- env:
- GH_TOKEN: ${{ github.token }}
- RELEASE_ID: ${{ inputs.release_id }}
- RELEASE_API_ID: ${{ steps.release.outputs.release_api_id }}
- MEDIA_SOURCES: ${{ inputs.media_sources }}
- run: |
- node <<'NODE'
- const { execFileSync } = require('node:child_process');
- const crypto = require('node:crypto');
- const fs = require('node:fs/promises');
- const path = require('node:path');
- const os = require('node:os');
-
- const repo = process.env.GITHUB_REPOSITORY;
- const token = process.env.GH_TOKEN;
- const releaseApiId = process.env.RELEASE_API_ID;
- const mediaSources = (() => {
- try { return JSON.parse(process.env.MEDIA_SOURCES || '{}') || {}; }
- catch { return {}; }
- })();
-
- function getMediaType(url) {
- const cleanUrl = String(url).split(/[?#]/)[0].toLowerCase();
- if (/^https:\/\/github\.com\/user-attachments\/assets\//.test(cleanUrl)) return 'githubAttachment';
- if (/^https:\/\/github\.com\/[^/]+\/[^/]+\/assets\/[^/]+\/[^/?#]+/.test(cleanUrl)) return 'githubAttachment';
- if (/\.(png|jpe?g|gif|webp|avif|svg)$/.test(cleanUrl)) return 'image';
- if (/\.(mp4|webm|ogg|mov|m4v)$/.test(cleanUrl)) return 'video';
- return null;
- }
-
- function safeAlt(label, fallback) {
- const value = String(label || fallback || 'Media').trim();
- return value.replace(/[\]\n\r]/g, '').replace(/^https?:\/\//i, '') || fallback || 'Media';
- }
-
- function sanitizeAssetName(name, fallbackUrl) {
- const fallbackName = (() => {
- try {
- const pathname = new URL(fallbackUrl).pathname;
- return decodeURIComponent(pathname.split('/').filter(Boolean).pop() || 'video.mp4');
- } catch {
- return 'video.mp4';
- }
- })();
- const rawName = String(name || fallbackName || 'video.mp4').trim();
- const ext = path.extname(rawName);
- const base = path.basename(rawName, ext) || crypto.createHash('sha1').update(fallbackUrl).digest('hex').slice(0, 10);
- const safeBase = base
- .replace(/[\\/:*?"<>|#%{}]/g, '-')
- .replace(/\s+/g, '-')
- .replace(/-+/g, '-')
- .replace(/^-|-$/g, '') || 'video';
- return `${safeBase}-preview.gif`;
- }
-
- function markdownImage(url, label) {
- return ``;
- }
-
- async function githubJson(method, apiPath, body) {
- const options = {
- method,
- headers: {
- Accept: 'application/vnd.github+json',
- Authorization: `Bearer ${token}`,
- 'X-GitHub-Api-Version': '2022-11-28',
- },
- };
- if (body !== undefined) {
- options.headers['Content-Type'] = 'application/json';
- options.body = JSON.stringify(body);
- }
- const response = await fetch(`https://api.github.com${apiPath}`, options);
- if (!response.ok) throw new Error(`${method} ${apiPath} failed: HTTP ${response.status}: ${await response.text()}`);
- if (response.status === 204) return null;
- return response.json();
- }
-
- async function uploadGifAsset(filePath, assetName) {
- const assets = await githubJson('GET', `/repos/${repo}/releases/${releaseApiId}/assets?per_page=100`);
- const existing = assets.find((asset) => asset.name === assetName);
- if (existing) await githubJson('DELETE', `/repos/${repo}/releases/assets/${existing.id}`);
-
- const buffer = await fs.readFile(filePath);
- const response = await fetch(`https://uploads.github.com/repos/${repo}/releases/${releaseApiId}/assets?name=${encodeURIComponent(assetName)}`, {
- method: 'POST',
- headers: {
- Accept: 'application/vnd.github+json',
- Authorization: `Bearer ${token}`,
- 'Content-Type': 'image/gif',
- 'Content-Length': String(buffer.byteLength),
- 'X-GitHub-Api-Version': '2022-11-28',
- },
- body: buffer,
- });
- if (!response.ok) throw new Error(`Upload ${assetName} failed: HTTP ${response.status}: ${await response.text()}`);
- const asset = await response.json();
- return asset.browser_download_url;
- }
-
- async function renderVideo(url, label) {
- const source = mediaSources[url] || { url, name: label };
- const downloadUrl = source.url || url;
- const assetName = sanitizeAssetName(source.name || label, url);
- const workDir = await fs.mkdtemp(path.join(os.tmpdir(), 'release-video-'));
- const inputPath = path.join(workDir, 'input-video');
- const outputPath = path.join(workDir, assetName);
-
- const response = await fetch(downloadUrl);
- if (!response.ok) throw new Error(`Download video failed for ${url}: HTTP ${response.status}`);
- await fs.writeFile(inputPath, Buffer.from(await response.arrayBuffer()));
-
- execFileSync('ffmpeg', [
- '-y',
- '-i', inputPath,
- '-t', '6',
- '-vf', 'fps=12,scale=1280:-1:flags=lanczos,format=rgb24,split[s0][s1];[s0]palettegen=max_colors=256:stats_mode=full[p];[s1][p]paletteuse=dither=sierra2_4a',
- outputPath,
- ], { stdio: 'inherit' });
-
- const gifUrl = await uploadGifAsset(outputPath, assetName);
- return `\n\n
\n\n`;
- }
-
- async function renderUrl(url, label) {
- const type = getMediaType(url);
- if (type === 'image') return markdownImage(url, label);
- if (type === 'githubAttachment') return `\n\n${url}\n\n`;
- if (type === 'video') return renderVideo(url, label);
- return null;
- }
-
- async function renderMarkdown(markdown) {
- let rendered = String(markdown || '');
- const markdownLinkPattern = /(!?)\[([^\]]*)\]\((https?:\/\/[^\s)]+)\)/g;
- const linkMatches = [...rendered.matchAll(markdownLinkPattern)];
- for (const match of linkMatches) {
- const [fullMatch, bang, label, url] = match;
- if (bang) continue;
- const replacement = await renderUrl(url, label);
- if (replacement) rendered = rendered.replace(fullMatch, replacement);
- }
-
- const bareUrlPattern = /(^|[\s>])(https?:\/\/[^\s<>)]+)/g;
- const bareMatches = [...rendered.matchAll(bareUrlPattern)];
- for (const match of bareMatches) {
- const [fullMatch, prefix, url] = match;
- const offset = match.index;
- if (prefix === '(' || rendered[offset - 1] === '(' || rendered[offset - 1] === '"') continue;
- const beforeUrl = rendered.slice(0, offset + prefix.length);
- const lastOpenParen = beforeUrl.lastIndexOf('(');
- const lastCloseParen = beforeUrl.lastIndexOf(')');
- if (lastOpenParen > lastCloseParen && /\[[^\]]*\]$/.test(beforeUrl.slice(0, lastOpenParen))) continue;
- const replacement = await renderUrl(url);
- if (replacement) rendered = rendered.replace(fullMatch, `${prefix}${replacement}`);
- }
-
- return rendered.replace(/\n{3,}/g, '\n\n').trim();
- }
-
- (async () => {
- const raw = await fs.readFile('release-body.md', 'utf8');
- const rendered = await renderMarkdown(raw);
- await fs.writeFile('release-body.md', `${rendered}\n`);
- })().catch((error) => {
- console.error(error);
- process.exit(1);
- });
- NODE
-
- - name: Update GitHub release body with media previews
- env:
- GH_TOKEN: ${{ github.token }}
- RELEASE_API_ID: ${{ steps.release.outputs.release_api_id }}
- run: |
- set -euo pipefail
- release_body="$(cat release-body.md)"
- jq -nc --arg body "${release_body}" '{ body: $body }' |
- gh api "repos/${GITHUB_REPOSITORY}/releases/${RELEASE_API_ID}" \
- --method PATCH \
- --input -
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 7f8b57bcc4..1823c90995 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -22,7 +22,7 @@ jobs:
strategy:
matrix:
- node-version: [22.18.0]
+ node-version: [22.22.3]
steps:
- uses: actions/checkout@v4
diff --git a/.github/workflows/v2-benchmark-tests.yml b/.github/workflows/v2-benchmark-tests.yml
index 54eaf768eb..24f878814b 100644
--- a/.github/workflows/v2-benchmark-tests.yml
+++ b/.github/workflows/v2-benchmark-tests.yml
@@ -23,7 +23,7 @@ jobs:
strategy:
matrix:
- node-version: [22.18.0]
+ node-version: [22.22.3]
steps:
- uses: actions/checkout@v4
diff --git a/.github/workflows/v2-core-tests.yml b/.github/workflows/v2-core-tests.yml
index 81a9382a02..47de660f71 100644
--- a/.github/workflows/v2-core-tests.yml
+++ b/.github/workflows/v2-core-tests.yml
@@ -33,10 +33,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- - name: Use Node.js 22.18.0
+ - name: Use Node.js 22.22.3
uses: actions/setup-node@v4
with:
- node-version: 22.18.0
+ node-version: 22.22.3
- name: 📥 Monorepo install
uses: ./.github/actions/pnpm-install
@@ -63,10 +63,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- - name: Use Node.js 22.18.0
+ - name: Use Node.js 22.22.3
uses: actions/setup-node@v4
with:
- node-version: 22.18.0
+ node-version: 22.22.3
- name: 📥 Monorepo install
uses: ./.github/actions/pnpm-install
@@ -102,10 +102,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- - name: Use Node.js 22.18.0
+ - name: Use Node.js 22.22.3
uses: actions/setup-node@v4
with:
- node-version: 22.18.0
+ node-version: 22.22.3
- name: 📥 Download all coverage artifacts
uses: actions/download-artifact@v4
diff --git a/.npmrc b/.npmrc
index 756b5df5d1..2f3a15b724 100644
--- a/.npmrc
+++ b/.npmrc
@@ -4,5 +4,5 @@ auto-install-peers=true
lockfile=true
# force use npmjs.org registry
registry=https://registry.npmjs.org/
-use-node-version=22.18.0
+use-node-version=22.22.3
save-prefix=''
diff --git a/Makefile b/Makefile
index e6aef817dc..0ee15cfe19 100644
--- a/Makefile
+++ b/Makefile
@@ -42,8 +42,9 @@ SCRATCH ?= /tmp
UNAME_S := $(shell uname -s)
-# set param statement_cache_size=1 to avoid query error `ERROR: cached plan must not change result type` after alter column type (modify field type)
-POSTGRES_PRISMA_DATABASE_URL ?= postgresql://teable:teable\@127.0.0.1:5432/teable?schema=public\&statement_cache_size=1
+# Disable prepared statement caching to avoid `ERROR: cached plan must not change result type`
+# after field schema changes in tests.
+POSTGRES_PRISMA_DATABASE_URL ?= postgresql://teable:teable\@127.0.0.1:5432/teable?schema=public\&statement_cache_size=0
# If the first make argument is "start", "stop"...
ifeq (docker.start,$(firstword $(MAKECMDGOALS)))
@@ -201,7 +202,7 @@ postgres.integration.test: docker.create.network
$(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) run -p 25432:5432 -d -T --no-deps --rm --name $$TEST_PG_CONTAINER_NAME teable-postgres; \
chmod +x scripts/wait-for; \
scripts/wait-for 127.0.0.1:25432 --timeout=15 -- echo 'pg database started successfully' && \
- export PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:25432/e2e_test_teable?schema=public\&statement_cache_size=1\&connection_limit=20 && \
+ export PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:25432/e2e_test_teable?schema=public\&statement_cache_size=0\&connection_limit=20 && \
make postgres.mode && \
pnpm -F "./packages/**" run build && \
pnpm g:test-e2e-cover && \
diff --git a/apps/nestjs-backend/package.json b/apps/nestjs-backend/package.json
index 9d69112059..5836d0e493 100644
--- a/apps/nestjs-backend/package.json
+++ b/apps/nestjs-backend/package.json
@@ -104,6 +104,7 @@
"eslint-config-next": "15.5.9",
"get-tsconfig": "4.7.3",
"istanbul-merge": "2.0.0",
+ "neverthrow": "8.2.0",
"npm-run-all2": "6.1.2",
"nyc": "15.1.0",
"pg-mem": "3.0.5",
@@ -167,7 +168,7 @@
"@opentelemetry/instrumentation-ioredis": "0.49.0",
"@opentelemetry/instrumentation-nestjs-core": "0.49.0",
"@opentelemetry/instrumentation-pg": "0.49.0",
- "@opentelemetry/instrumentation-pino": "0.49.0",
+ "@opentelemetry/instrumentation-pino": "0.54.0",
"@opentelemetry/instrumentation-runtime-node": "0.24.0",
"@opentelemetry/resources": "2.0.1",
"@opentelemetry/sdk-node": "0.201.1",
@@ -228,6 +229,7 @@
"keyv": "4.5.4",
"knex": "3.1.0",
"lodash": "4.17.21",
+ "markdown-it": "14.1.0",
"mime-types": "2.1.35",
"minio": "7.1.3",
"ms": "2.1.3",
diff --git a/apps/nestjs-backend/src/app.module.ts b/apps/nestjs-backend/src/app.module.ts
index 714ed3783b..71e7f9a03e 100644
--- a/apps/nestjs-backend/src/app.module.ts
+++ b/apps/nestjs-backend/src/app.module.ts
@@ -10,10 +10,12 @@ import { ConfigModule } from './configs/config.module';
import { AccessTokenModule } from './features/access-token/access-token.module';
import { AggregationOpenApiModule } from './features/aggregation/open-api/aggregation-open-api.module';
import { AiModule } from './features/ai/ai.module';
+import { AirtableImportModule } from './features/airtable-import/airtable-import.module';
import { AttachmentsModule } from './features/attachments/attachments.module';
import { AuthModule } from './features/auth/auth.module';
import { BaseModule } from './features/base/base.module';
import { BaseNodeModule } from './features/base-node/base-node.module';
+import { BaseShareModule } from './features/base-share/base-share.module';
import { BuiltinAssetsInitModule } from './features/builtin-assets-init';
import { CanaryModule } from './features/canary';
import { ChatModule } from './features/chat/chat.module';
@@ -40,7 +42,6 @@ import { PluginPanelModule } from './features/plugin-panel/plugin-panel.module';
import { SelectionModule } from './features/selection/selection.module';
import { AdminOpenApiModule } from './features/setting/open-api/admin-open-api.module';
import { SettingOpenApiModule } from './features/setting/open-api/setting-open-api.module';
-import { BaseShareModule } from './features/base-share/base-share.module';
import { ShareModule } from './features/share/share.module';
import { SpaceModule } from './features/space/space.module';
import { TemplateOpenApiModule } from './features/template/template-open-api.module';
@@ -88,6 +89,7 @@ export const appModules = {
NotificationModule,
AccessTokenModule,
ImportOpenApiModule,
+ AirtableImportModule,
ExportOpenApiModule,
PinModule,
AdminOpenApiModule,
diff --git a/apps/nestjs-backend/src/cache/types.ts b/apps/nestjs-backend/src/cache/types.ts
index 942cc59623..9d02293e55 100644
--- a/apps/nestjs-backend/src/cache/types.ts
+++ b/apps/nestjs-backend/src/cache/types.ts
@@ -34,9 +34,11 @@ export interface ICacheStore {
[key: `waitlist:invite-code:${string}`]: number;
[key: `send-mail-rate-limit:${string}`]: boolean;
[key: `oauth:token-rate:${string}:${string}`]: number;
- [key: `automation:email:rate:${string}:${number}`]: number;
+ [key: `email:send:rate:${string}:${number}`]: number;
[key: `automation:email-att:${string}`]: string[];
[key: `automation:fail-notify-count:${string}`]: number;
+ // Watchdog round-robin scan cursor per status (staleAt stored as ISO string).
+ [key: `automation:orphan-cursor:${string}`]: { staleAt: string; key: string };
// Distributed lock keys
[key: `lock:${string}`]: string;
[key: `import:result:manifest:${string}`]: {
diff --git a/apps/nestjs-backend/src/configs/threshold.config.ts b/apps/nestjs-backend/src/configs/threshold.config.ts
index 83a550b690..215b481a9b 100644
--- a/apps/nestjs-backend/src/configs/threshold.config.ts
+++ b/apps/nestjs-backend/src/configs/threshold.config.ts
@@ -20,6 +20,9 @@ export const thresholdConfig = registerAs('threshold', () => ({
bigTransactionTimeout: Number(
process.env.BIG_TRANSACTION_TIMEOUT ?? 10 * 60 * 1000 /* 10 mins */
),
+ // DB statement_timeout (ms) for the search query, so a slow / full-scan search is canceled
+ // and its connection released instead of being held for minutes. Tune via SEARCH_TIMEOUT.
+ searchTimeout: Number(process.env.SEARCH_TIMEOUT ?? 15_000 /* 15s */),
automationGap: Number(process.env.AUTOMATION_GAP ?? 200),
maxAttachmentUploadSize: Number(process.env.MAX_ATTACHMENT_UPLOAD_SIZE ?? Infinity),
maxOpenapiAttachmentUploadSize: Number(
diff --git a/apps/nestjs-backend/src/db-provider/filter-query/__tests__/numeric-text-filter.spec.ts b/apps/nestjs-backend/src/db-provider/filter-query/__tests__/numeric-text-filter.spec.ts
new file mode 100644
index 0000000000..62ee78595c
--- /dev/null
+++ b/apps/nestjs-backend/src/db-provider/filter-query/__tests__/numeric-text-filter.spec.ts
@@ -0,0 +1,68 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import {
+ CellValueType,
+ DriverClient,
+ FieldType,
+ SingleLineTextFieldCore,
+ is,
+ isNot,
+} from '@teable/core';
+import type { FieldCore, IFilter } from '@teable/core';
+import knex from 'knex';
+import type { IDbProvider } from '../../db.provider.interface';
+import { FilterQueryPostgres } from '../postgres/filter-query.postgres';
+
+const knexBuilder = knex({ client: 'pg' });
+const dbProviderStub = { driver: DriverClient.Pg } as unknown as IDbProvider;
+
+function createTextField(id: string, dbFieldName: string): SingleLineTextFieldCore {
+ const field = new SingleLineTextFieldCore();
+ field.id = id;
+ field.name = id;
+ field.dbFieldName = dbFieldName;
+ field.type = FieldType.SingleLineText;
+ field.options = SingleLineTextFieldCore.defaultOptions();
+ field.cellValueType = CellValueType.String;
+ field.isMultipleCellValue = false;
+ field.isLookup = false;
+ field.updateDbFieldType();
+ return field;
+}
+
+function buildSql(field: FieldCore, filter: IFilter): string {
+ const qb = knexBuilder('main_table as main');
+ new FilterQueryPostgres(qb, { [field.id]: field }, filter, undefined, dbProviderStub, {
+ selectionMap: new Map([[field.id, `"main"."${field.dbFieldName}"`]]),
+ }).appendQueryBuilder();
+ return qb.toQuery().replace(/\s+/g, ' ');
+}
+
+describe('numeric value on a text field filter', () => {
+ // A numeric-looking value (e.g. a tracking number) arrives as a JS number per
+ // literalValueSchema. Rendered to SQL it must be a quoted string, otherwise
+ // Postgres rejects "text = bigint".
+ const trackingNumber = 872985557030;
+
+ it('renders "is" as a quoted string literal, not a bigint', () => {
+ const field = createTextField('fld_text', 'tracking_number');
+ const filter: IFilter = {
+ conjunction: 'and',
+ filterSet: [{ fieldId: field.id, operator: is.value, value: trackingNumber }],
+ };
+
+ const sql = buildSql(field, filter);
+ expect(sql).toContain(`"main"."tracking_number" = '${trackingNumber}'`);
+ expect(sql).not.toMatch(new RegExp(`=\\s*${trackingNumber}(?!')`));
+ });
+
+ it('renders "isNot" as a quoted string literal, not a bigint', () => {
+ const field = createTextField('fld_text', 'tracking_number');
+ const filter: IFilter = {
+ conjunction: 'and',
+ filterSet: [{ fieldId: field.id, operator: isNot.value, value: trackingNumber }],
+ };
+
+ const sql = buildSql(field, filter);
+ expect(sql).toContain(`"main"."tracking_number" IS DISTINCT FROM '${trackingNumber}'`);
+ });
+});
diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/string-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/string-cell-value-filter.adapter.ts
index be892185d5..99d64df3b7 100644
--- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/string-cell-value-filter.adapter.ts
+++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/string-cell-value-filter.adapter.ts
@@ -1,5 +1,4 @@
import {
- CellValueType,
isFieldReferenceValue,
type IFieldReferenceValue,
type IFilterOperator,
@@ -22,8 +21,10 @@ export class StringCellValueFilterAdapter extends CellValueFilterPostgres {
builderClient.whereRaw(`${this.tableColumnRef} = ${ref}`);
return builderClient;
}
- const parseValue = this.field.cellValueType === CellValueType.Number ? Number(value) : value;
- builderClient.whereRaw(`${this.tableColumnRef} = ?`, [parseValue]);
+ // Column is text; coerce numeric/boolean literals to string so the comparison stays
+ // text = text. Otherwise a numeric value renders as a bigint literal and Postgres
+ // rejects it with "operator does not exist: text = bigint".
+ builderClient.whereRaw(`${this.tableColumnRef} = ?`, [String(value)]);
return builderClient;
}
@@ -33,14 +34,12 @@ export class StringCellValueFilterAdapter extends CellValueFilterPostgres {
value: ILiteralValue | IFieldReferenceValue,
_dbProvider: IDbProvider
): Knex.QueryBuilder {
- const { cellValueType } = this.field;
if (isFieldReferenceValue(value)) {
const ref = this.resolveFieldReference(value);
builderClient.whereRaw(`${this.tableColumnRef} IS DISTINCT FROM ${ref}`);
return builderClient;
}
- const parseValue = cellValueType === CellValueType.Number ? Number(value) : value;
- builderClient.whereRaw(`${this.tableColumnRef} IS DISTINCT FROM ?`, [parseValue]);
+ builderClient.whereRaw(`${this.tableColumnRef} IS DISTINCT FROM ?`, [String(value)]);
return builderClient;
}
diff --git a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.spec.ts b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.spec.ts
index 069a3cf712..c932ed8176 100644
--- a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.spec.ts
+++ b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.spec.ts
@@ -22,6 +22,17 @@ const buildDateField = (): IFieldInstance =>
},
}) as IFieldInstance;
+const buildMultipleSelectField = (): IFieldInstance =>
+ ({
+ id: 'fldMultiSelect0001',
+ dbFieldName: 'Tags',
+ cellValueType: CellValueType.String,
+ isMultipleCellValue: true,
+ isStructuredCellValue: false,
+ type: FieldType.MultipleSelect,
+ options: {},
+ }) as IFieldInstance;
+
describe('SearchQueryPostgres', () => {
const db = knex({ client: 'pg' });
@@ -52,4 +63,19 @@ describe('SearchQueryPostgres', () => {
const compiled = builder.getQuery()?.toSQL();
expect(compiled?.sql).toBe('FALSE');
});
+
+ it('matches multipleSelect as a text cast so the gin_trgm index can be used', () => {
+ const field = buildMultipleSelectField();
+ const builder = new SearchQueryPostgres(
+ db.queryBuilder(),
+ field,
+ ['Beta', 'fldMultiSelect0001'],
+ []
+ );
+
+ const compiled = builder.getQuery()?.toSQL();
+ expect(compiled?.sql).toContain('("Tags")::text ILIKE');
+ expect(compiled?.sql).not.toContain('jsonb_array_elements');
+ expect(compiled?.bindings).toEqual(['%Beta%']);
+ });
});
diff --git a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts
index 53283be572..20d514ae9c 100644
--- a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts
+++ b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts
@@ -97,9 +97,11 @@ export class SearchQueryPostgres extends SearchQueryAbstract {
case CellValueType.String: {
if (isStructuredCellValue) {
return this.multipleJson();
- } else {
- return this.multipleText();
}
+ if (field.type === FieldType.MultipleSelect) {
+ return this.multipleSelectText();
+ }
+ return this.multipleText();
}
case CellValueType.DateTime: {
return this.multipleDate();
@@ -180,6 +182,16 @@ export class SearchQueryPostgres extends SearchQueryAbstract {
);
}
+ // multipleSelect stores a plain string[] of option names. Match the whole cell as text so the
+ // predicate is sargable against the gin_trgm index (built on the same ""::text expression)
+ // instead of a jsonb_array_elements + regex subquery that cannot use the index. Trades negligible
+ // precision (JSON brackets/quotes become matchable) for index usage.
+ protected multipleSelectText() {
+ const { search, knex } = this;
+ const escapedSearchValue = escapeLikeWildcards(search[0]);
+ return knex.raw(`(${this.fieldName})::text ILIKE ? ESCAPE '\\'`, [`%${escapedSearchValue}%`]);
+ }
+
protected multipleNumber() {
const { search, knex } = this;
const searchValue = search[0];
diff --git a/apps/nestjs-backend/src/event-emitter/event-job/event-job.module.ts b/apps/nestjs-backend/src/event-emitter/event-job/event-job.module.ts
index 51ec559dee..0ceda5de7e 100644
--- a/apps/nestjs-backend/src/event-emitter/event-job/event-job.module.ts
+++ b/apps/nestjs-backend/src/event-emitter/event-job/event-job.module.ts
@@ -15,6 +15,9 @@ const queueOptions: NestWorkerOptions = {
},
};
+const isTestOrCI = process.env.CI || process.env.NODE_ENV === 'test' || process.env.VITEST;
+const CONDITIONAL_MODULE_TIMEOUT = isTestOrCI ? 60000 : 5000;
+
@Module({
imports: [ConfigModule],
})
@@ -26,11 +29,13 @@ export class EventJobModule {
name,
...queueOptions,
}),
- (env) => Boolean(env.BACKEND_CACHE_REDIS_URI)
+ (env) => Boolean(env.BACKEND_CACHE_REDIS_URI),
+ { timeout: CONDITIONAL_MODULE_TIMEOUT }
),
ConditionalModule.registerWhen(
FallbackQueueModule.registerQueue(name),
- (env) => !env.BACKEND_CACHE_REDIS_URI
+ (env) => !env.BACKEND_CACHE_REDIS_URI,
+ { timeout: CONDITIONAL_MODULE_TIMEOUT }
),
]);
diff --git a/apps/nestjs-backend/src/event-emitter/events/event.enum.ts b/apps/nestjs-backend/src/event-emitter/events/event.enum.ts
index 6588e55e74..688450e277 100644
--- a/apps/nestjs-backend/src/event-emitter/events/event.enum.ts
+++ b/apps/nestjs-backend/src/event-emitter/events/event.enum.ts
@@ -51,6 +51,7 @@ export enum Events {
SHARED_VIEW_CREATE = 'shared.view.create',
SHARED_VIEW_DELETE = 'shared.view.delete',
SHARED_VIEW_UPDATE = 'shared.view.update',
+ SHARED_VIEW_REFRESH = 'shared.view.refresh',
USER_SIGNIN = 'user.signin',
USER_SIGNUP = 'user.signup',
@@ -67,6 +68,20 @@ export enum Events {
COLLABORATOR_DELETE = 'collaborator.delete',
COLLABORATOR_UPDATE = 'collaborator.update',
+ // Base-scope collaborator audit actions (parallel to the generic COLLABORATOR_*
+ // business events above, which are kept for internal pub/sub). Future space-level
+ // audit can mirror this with SPACE_COLLABORATOR_*.
+ BASE_COLLABORATOR_CREATE = 'base.collaborator.create',
+ BASE_COLLABORATOR_DELETE = 'base.collaborator.delete',
+ BASE_COLLABORATOR_UPDATE = 'base.collaborator.update',
+
+ // Base/Node share lifecycle (covers both node-scoped and base-wide shares;
+ // payload.type distinguishes 'node' | 'base').
+ BASE_SHARE_CREATE = 'base.share.create',
+ BASE_SHARE_UPDATE = 'base.share.update',
+ BASE_SHARE_DELETE = 'base.share.delete',
+ BASE_SHARE_REFRESH = 'base.share.refresh',
+
BASE_FOLDER_CREATE = 'base.folder.create',
BASE_FOLDER_DELETE = 'base.folder.delete',
BASE_FOLDER_UPDATE = 'base.folder.update',
@@ -98,6 +113,7 @@ export enum Events {
// completion point — past `lastChunk`, error-file write, and presence cleanup —
// instead of racing against per-chunk audit emits.
TABLE_IMPORT_FINISH = 'table.import.finish',
+ V2_TABLE_IMPORT_FINISH = 'v2.table.import.finish',
// Composite-operation terminal signals. Currently only e2e tests subscribe; reserved
// for future use by notifications / webhooks / billing. Each fires after the synchronous
diff --git a/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts
index 8030d2b54d..8e2dfd27fb 100644
--- a/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts
+++ b/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts
@@ -3,12 +3,10 @@ import { OnEvent } from '@nestjs/event-emitter';
import type { ITableActionKey, IGridColumn, IViewActionKey } from '@teable/core';
import { getActionTriggerChannel, OpName } from '@teable/core';
import { isEmpty } from 'lodash';
-import { ClsService } from 'nestjs-cls';
import { match } from 'ts-pattern';
-import { getV2CreateTableLegacyEventsFlag } from '../../features/v2/v2-create-table-compat.constants';
import { ShareDbService } from '../../share-db/share-db.service';
-import type { IClsStore } from '../../types/cls';
import type {
+ IChangeRecord,
RecordCreateEvent,
RecordDeleteEvent,
RecordUpdateEvent,
@@ -19,6 +17,26 @@ import type {
} from '../events';
import { Events } from '../events';
+const collectChangedRecordFieldIds = (record: IChangeRecord | IChangeRecord[]): string[] => {
+ const records = Array.isArray(record) ? record : [record];
+ const fieldIds = new Set();
+ for (const changeRecord of records) {
+ for (const fieldId of Object.keys(changeRecord?.fields ?? {})) {
+ fieldIds.add(fieldId);
+ }
+ }
+ return [...fieldIds];
+};
+
+const schemaRefreshFieldProperties = new Set(['type', 'options', 'dbFieldType']);
+
+const hasSchemaRefreshFieldChange = (field: FieldUpdateEvent['payload']['field']): boolean => {
+ const fields = Array.isArray(field) ? field : [field];
+ return fields.some((changeField) =>
+ Object.keys(changeField).some((key) => schemaRefreshFieldProperties.has(key))
+ );
+};
+
type IViewEvent = ViewUpdateEvent;
type IRecordEvent = RecordCreateEvent | RecordDeleteEvent | RecordUpdateEvent;
type IListenerEvent =
@@ -37,10 +55,7 @@ export interface IActionTriggerData {
export class ActionTriggerListener {
private readonly logger = new Logger(ActionTriggerListener.name);
- constructor(
- private readonly shareDbService: ShareDbService,
- private readonly cls: ClsService
- ) {}
+ constructor(private readonly shareDbService: ShareDbService) {}
@OnEvent(Events.TABLE_VIEW_UPDATE, { async: true })
@OnEvent(Events.TABLE_FIELD_UPDATE, { async: true })
@@ -48,13 +63,6 @@ export class ActionTriggerListener {
@OnEvent(Events.TABLE_FIELD_DELETE, { async: true })
@OnEvent('table.record.*', { async: true })
private async listener(listenerEvent: IListenerEvent): Promise {
- if (
- getV2CreateTableLegacyEventsFlag(this.cls) &&
- (this.isTableFieldCreateEvent(listenerEvent) || this.isTableRecordEvent(listenerEvent))
- ) {
- return;
- }
-
// Handling table view update events
if (this.isTableViewUpdateEvent(listenerEvent)) {
await this.handleTableViewUpdate(listenerEvent as ViewUpdateEvent);
@@ -141,17 +149,21 @@ export class ActionTriggerListener {
const { tableId } = event.payload;
const buffer = match(event)
- .returnType()
- .with({ name: Events.TABLE_RECORD_CREATE }, () => ['addRecord'])
- .with({ name: Events.TABLE_RECORD_UPDATE }, () => ['setRecord'])
- .with({ name: Events.TABLE_RECORD_DELETE }, () => ['deleteRecord'])
+ .returnType()
+ .with({ name: Events.TABLE_RECORD_CREATE }, () => [{ actionKey: 'addRecord' as const }])
+ .with({ name: Events.TABLE_RECORD_UPDATE }, (updateEvent) => [
+ {
+ actionKey: 'setRecord' as const,
+ // changed cell field ids, letting field-aware listeners skip
+ // refreshes for irrelevant edits (same contract as the v2 emitter)
+ payload: { fieldIds: collectChangedRecordFieldIds(updateEvent.payload.record) },
+ },
+ ])
+ .with({ name: Events.TABLE_RECORD_DELETE }, () => [{ actionKey: 'deleteRecord' as const }])
.otherwise(() => []);
if (!isEmpty(buffer)) {
- this.emitActionTrigger(
- tableId,
- buffer.map((actionKey) => ({ actionKey }))
- );
+ this.emitActionTrigger(tableId, buffer);
}
}
@@ -178,9 +190,11 @@ export class ActionTriggerListener {
}
private isValidFieldUpdateOperation(event: FieldUpdateEvent): boolean | undefined {
- const propertyKeys = ['options', 'dbFieldType'];
const { propertyKey } = event.context.opMeta || {};
- return propertyKeys.includes(propertyKey as string);
+ return (
+ schemaRefreshFieldProperties.has(propertyKey as string) ||
+ hasSchemaRefreshFieldChange(event.payload.field)
+ );
}
private isTableRecordEvent(event: IListenerEvent): boolean {
diff --git a/apps/nestjs-backend/src/features/access-token/access-token.service.ts b/apps/nestjs-backend/src/features/access-token/access-token.service.ts
index 2e3460af0b..70c1e24b44 100644
--- a/apps/nestjs-backend/src/features/access-token/access-token.service.ts
+++ b/apps/nestjs-backend/src/features/access-token/access-token.service.ts
@@ -142,6 +142,18 @@ export class AccessTokenService {
@Audit({
action: Events.ACCESS_TOKEN_CREATE,
resourceId: (input: { userId?: string }, ctx) => input.userId ?? ctx.cls.get('user.id')!,
+ userId: (input: { userId?: string }, ctx) => input.userId ?? ctx.cls.get('user.id'),
+ // Record the token's settings so the audit row shows what access was granted. NEVER the secret:
+ // the token `sign` is generated server-side and is not part of the input, so this is safe.
+ params: (input: CreateAccessTokenRo & { clientId?: string }) => ({
+ name: input.name,
+ description: input.description,
+ scopes: input.scopes,
+ spaceIds: input.spaceIds,
+ baseIds: input.baseIds,
+ expiredTime: input.expiredTime,
+ hasFullAccess: input.hasFullAccess,
+ }),
emit: true,
})
async createAccessToken(
diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts
index a016d80940..5c10c744cd 100644
--- a/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts
+++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts
@@ -1,7 +1,7 @@
import type { IFilter, IGroup, ISortItem, StatisticsFunc } from '@teable/core';
import type {
IAggregationField,
- IQueryBaseRo,
+ IRowCountRo,
IRawAggregationValue,
IRawAggregations,
IRawRowCountValue,
@@ -60,7 +60,7 @@ export interface IAggregationService {
* @param queryRo - Query parameters for filtering
* @returns Promise - The row count result
*/
- performRowCount(tableId: string, queryRo: IQueryBaseRo): Promise;
+ performRowCount(tableId: string, queryRo: IRowCountRo): Promise;
/**
* Get field data for a table
diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts
index c6f1d8ec10..02709a4727 100644
--- a/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts
+++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts
@@ -1,5 +1,4 @@
import { Injectable, Logger } from '@nestjs/common';
-import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library';
import {
CellValueType,
HttpErrorCode,
@@ -17,7 +16,7 @@ import { PrismaService } from '@teable/db-main-prisma';
import { StatisticsFunc } from '@teable/openapi';
import type {
IAggregationField,
- IQueryBaseRo,
+ IRowCountRo,
IRawAggregationValue,
IRawAggregations,
IRawRowCountValue,
@@ -501,7 +500,7 @@ export class AggregationService implements IAggregationService {
* @returns Promise - The row count result
* @throws NotImplementedException - This method is not yet implemented
*/
- async performRowCount(tableId: string, queryRo: IQueryBaseRo): Promise {
+ async performRowCount(tableId: string, queryRo: IRowCountRo): Promise {
const {
viewId,
ignoreViewQuery,
@@ -509,6 +508,7 @@ export class AggregationService implements IAggregationService {
filterLinkCellSelected,
selectedRecordIds,
search,
+ projection,
} = queryRo;
// Retrieve the current user's ID to build user-related query conditions
const currentUserId = this.cls.get('user.id');
@@ -534,6 +534,7 @@ export class AggregationService implements IAggregationService {
filterLinkCellSelected,
selectedRecordIds,
search,
+ projection,
withUserId: currentUserId,
viewId: queryRo?.viewId,
});
@@ -560,6 +561,7 @@ export class AggregationService implements IAggregationService {
filterLinkCellSelected?: IGetRecordsRo['filterLinkCellSelected'];
selectedRecordIds?: IGetRecordsRo['selectedRecordIds'];
search?: [string, string?, boolean?];
+ projection?: string[];
withUserId?: string;
viewId?: string;
}) {
@@ -572,6 +574,7 @@ export class AggregationService implements IAggregationService {
filterLinkCellSelected,
selectedRecordIds,
search,
+ projection,
withUserId,
viewId,
} = params;
@@ -605,10 +608,17 @@ export class AggregationService implements IAggregationService {
);
if (search && search[2]) {
+ const enabledFieldIds = wrap.enabledFieldIds;
+ const searchProjection = projection
+ ? enabledFieldIds
+ ? projection.filter((fieldId) => enabledFieldIds.includes(fieldId))
+ : projection
+ : enabledFieldIds;
const searchFields = await this.recordService.getSearchFields(
fieldInstanceMap,
search,
- viewId
+ viewId,
+ searchProjection
);
const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId);
qb.where((builder) => {
@@ -1014,8 +1024,15 @@ export class AggregationService implements IAggregationService {
this.logger.debug('getRecordIndexBySearchOrder sql: %s', sql);
+ const searchTimeout = this.thresholdConfig.searchTimeout;
+
try {
return await this.withDataPrismaTransaction(tableId, async (prisma) => {
+ // Bound the search at the DB level: a short / CJK term can defeat the pg_trgm index and
+ // degrade to a full-table scan. SET LOCAL statement_timeout makes Postgres cancel the
+ // statement and release the pooled connection, instead of the client abandoning the
+ // request while the query keeps running and starves the connection pool.
+ await prisma.$executeRawUnsafe(`SET LOCAL statement_timeout = ${searchTimeout}`);
const result = await prisma.$queryRawUnsafe<{ __id: string; fieldId: string }[]>(sql);
// no result found
@@ -1080,18 +1097,52 @@ export class AggregationService implements IAggregationService {
recordId: item.__id,
};
});
+ // statement_timeout (above) is the real cap; give the JS-side tx timer headroom so
+ // Postgres cancels the statement first, instead of the data-tx default timeout firing early.
});
} catch (error) {
- if (error instanceof PrismaClientKnownRequestError && error.code === 'P2028') {
- throw new CustomHttpException(`${error.message}`, HttpErrorCode.REQUEST_TIMEOUT, {
- localization: {
- i18nKey: 'httpErrors.aggregation.searchTimeOut',
- },
- });
+ if (this.isSearchTimeoutError(error)) {
+ throw new CustomHttpException(
+ `${(error as Error).message}`,
+ HttpErrorCode.REQUEST_TIMEOUT,
+ {
+ localization: {
+ i18nKey: 'httpErrors.aggregation.searchTimeOut',
+ },
+ }
+ );
}
throw error;
}
}
+
+ /**
+ * Detects a search timeout from any of the layers involved. The search runs on the data-db
+ * Prisma client, whose error classes come from a *separate* generated runtime, so cross-package
+ * `instanceof` (PrismaClientKnownRequestError / TimeoutHttpException) is unreliable here — we
+ * duck-type on the error shape instead:
+ * - Postgres cancels the statement via `SET LOCAL statement_timeout` → SQLSTATE 57014,
+ * - Prisma's interactive-transaction timeout → code P2028,
+ * - the data-prisma proxy converts P2028 into a TimeoutHttpException → code 'request_timeout'.
+ * The DB-level cancellation (57014) is what actually releases the pooled connection.
+ */
+ private isSearchTimeoutError(error: unknown): boolean {
+ if (typeof error !== 'object' || error === null) {
+ return false;
+ }
+ const err = error as { code?: unknown; meta?: unknown; message?: unknown };
+ const pgErrorCode =
+ typeof err.meta === 'object' && err.meta !== null
+ ? (err.meta as { code?: unknown }).code
+ : undefined;
+ const message = typeof err.message === 'string' ? err.message : '';
+ return (
+ err.code === 'P2028' ||
+ err.code === 'request_timeout' ||
+ pgErrorCode === '57014' ||
+ /canceling statement due to statement timeout|Transaction already closed/i.test(message)
+ );
+ }
async getRecordIndex(tableId: string, queryRo: IRecordIndexRo): Promise {
const { recordId } = queryRo;
diff --git a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts
index 7a152b2a86..86beba965b 100644
--- a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts
+++ b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts
@@ -18,10 +18,10 @@ import {
groupPointsRoSchema,
IAggregationRo,
IGroupPointsRo,
- IQueryBaseRo,
+ IRowCountRo,
searchCountRoSchema,
ISearchCountRo,
- queryBaseSchema,
+ rowCountRoSchema,
ICalendarDailyCollectionRo,
ISearchIndexByQueryRo,
searchIndexByQueryRoSchema,
@@ -115,7 +115,7 @@ export class AggregationOpenApiController {
@Permissions('table|read')
async getRowCount(
@Param('tableId') tableId: string,
- @Query(new ZodValidationPipe(queryBaseSchema), TqlPipe) query?: IQueryBaseRo
+ @Query(new ZodValidationPipe(rowCountRoSchema), TqlPipe) query?: IRowCountRo
): Promise {
return await this.getAggregationWithCache('row_count', tableId, query, () =>
this.aggregationOpenApiService.getRowCount(tableId, query)
diff --git a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts
index aa28ad35a8..24145164f1 100644
--- a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts
+++ b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts
@@ -9,7 +9,7 @@ import type {
ICalendarDailyCollectionVo,
IGroupPointsRo,
IGroupPointsVo,
- IQueryBaseRo,
+ IRowCountRo,
IRowCountVo,
ISearchCountRo,
IRecordIndexRo,
@@ -69,7 +69,7 @@ export class AggregationOpenApiService {
return { aggregations: result?.aggregations };
}
- async getRowCount(tableId: string, query: IQueryBaseRo = {}): Promise {
+ async getRowCount(tableId: string, query: IRowCountRo = {}): Promise {
const result = await this.aggregationService.performRowCount(tableId, query);
return {
rowCount: result.rowCount,
diff --git a/apps/nestjs-backend/src/features/ai/util.ts b/apps/nestjs-backend/src/features/ai/util.ts
index 60cb373dd0..17187eca81 100644
--- a/apps/nestjs-backend/src/features/ai/util.ts
+++ b/apps/nestjs-backend/src/features/ai/util.ts
@@ -88,28 +88,6 @@ const createOpenAICompatibleWrapper = (
});
};
-const createClaudeCodeWrapper = (
- options: Parameters[0]
-): ReturnType => {
- const baseFetch = createFixingFetch();
- const claudeCodeDefaultUa = 'claude-cli/2.1.71 (external, cli)';
- return createAnthropic({
- ...options,
- fetch: async (input, init) => {
- const initHeaders = (init?.headers ?? {}) as Record;
- const ua = initHeaders['user-agent'];
- return baseFetch(input, {
- ...init,
- headers: {
- ...init?.headers,
- // eslint-disable-next-line @typescript-eslint/naming-convention
- 'user-agent': ua?.includes('claude-cli') ? ua : claudeCodeDefaultUa,
- },
- });
- },
- });
-};
-
export const modelProviders = {
[LLMProviderType.OPENAI]: createOpenAI,
[LLMProviderType.ANTHROPIC]: createAnthropic,
@@ -127,7 +105,6 @@ export const modelProviders = {
[LLMProviderType.AMAZONBEDROCK]: createAmazonBedrock,
[LLMProviderType.OPENROUTER]: createOpenRouter,
[LLMProviderType.OPENAI_COMPATIBLE]: createOpenAICompatibleWrapper,
- [LLMProviderType.CLAUDE_CODE]: createClaudeCodeWrapper,
// AI_GATEWAY is handled separately in ai.service.ts using createGateway from 'ai'
} as const;
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-api.client.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-api.client.ts
new file mode 100644
index 0000000000..4366454fab
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-api.client.ts
@@ -0,0 +1,168 @@
+import type {
+ IAirtableBaseItem,
+ IAirtableBaseSchemaResponse,
+ IAirtableListBasesResponse,
+ IAirtableListRecordsResponse,
+ IAirtableRecord,
+ IAirtableTable,
+} from './airtable.types';
+
+const airtableApiBaseUrl = 'https://api.airtable.com/v0';
+// Airtable allows 5 requests/second per base; stay slightly under it.
+const minRequestIntervalMs = 220;
+// Airtable documents a 30s penalty after a 429 response.
+const rateLimitWaitMs = 30_000;
+const maxRetries = 3;
+const recordsPageSize = 100;
+
+export class AirtableApiError extends Error {
+ constructor(
+ message: string,
+ public readonly status: number,
+ public readonly type?: string
+ ) {
+ super(message);
+ this.name = 'AirtableApiError';
+ }
+}
+
+/** The list-records pagination iterator expired (HTTP 422); listing must restart. */
+export class AirtableIteratorExpiredError extends AirtableApiError {
+ constructor() {
+ super('Airtable records iterator expired', 422, 'LIST_RECORDS_ITERATOR_NOT_AVAILABLE');
+ this.name = 'AirtableIteratorExpiredError';
+ }
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * The token is fetched per request so integration-backed imports keep working
+ * past the ~60 minute Airtable OAuth token lifetime (the provider refreshes
+ * server-side).
+ */
+export type IAirtableAccessTokenProvider = () => Promise | string;
+
+export class AirtableApiClient {
+ private lastRequestAt = 0;
+
+ constructor(private readonly getAccessToken: IAirtableAccessTokenProvider) {}
+
+ async listBases(): Promise {
+ const bases: IAirtableBaseItem[] = [];
+ let offset: string | undefined;
+ do {
+ const query = offset ? `?offset=${encodeURIComponent(offset)}` : '';
+ const res = await this.request(`/meta/bases${query}`);
+ bases.push(...res.bases);
+ offset = res.offset;
+ } while (offset);
+ return bases;
+ }
+
+ async getBaseSchema(airtableBaseId: string): Promise {
+ const res = await this.request(
+ `/meta/bases/${encodeURIComponent(airtableBaseId)}/tables`
+ );
+ return res.tables;
+ }
+
+ /**
+ * Yields pages of records (100 per page). Throws AirtableIteratorExpiredError
+ * when the pagination iterator expires; the caller may restart the listing
+ * and deduplicate already-processed records by id.
+ */
+ async *listRecords(airtableBaseId: string, tableId: string): AsyncGenerator {
+ let offset: string | undefined;
+ do {
+ const params = new URLSearchParams({
+ pageSize: String(recordsPageSize),
+ returnFieldsByFieldId: 'true',
+ cellFormat: 'json',
+ });
+ if (offset) {
+ params.set('offset', offset);
+ }
+ const res = await this.request(
+ `/${encodeURIComponent(airtableBaseId)}/${encodeURIComponent(tableId)}?${params.toString()}`
+ );
+ if (res.records.length > 0) {
+ yield res.records;
+ }
+ offset = res.offset;
+ } while (offset);
+ }
+
+ private async throttle() {
+ const wait = this.lastRequestAt + minRequestIntervalMs - Date.now();
+ if (wait > 0) {
+ await sleep(wait);
+ }
+ this.lastRequestAt = Date.now();
+ }
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+ private async request(path: string): Promise {
+ let attempt = 0;
+ // Retry budget covers both rate-limit waits and transient network errors.
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ await this.throttle();
+ const accessToken = await this.getAccessToken();
+ let response: Response;
+ try {
+ response = await fetch(`${airtableApiBaseUrl}${path}`, {
+ headers: { Authorization: `Bearer ${accessToken}` },
+ });
+ } catch (e) {
+ if (attempt >= maxRetries) {
+ throw new AirtableApiError(
+ `Failed to reach the Airtable API: ${e instanceof Error ? e.message : 'network error'}`,
+ 0
+ );
+ }
+ await sleep(1000 * 2 ** attempt);
+ attempt++;
+ continue;
+ }
+
+ if (response.ok) {
+ return (await response.json()) as T;
+ }
+
+ const errorBody = await this.parseError(response);
+ if (response.status === 429 && attempt < maxRetries) {
+ await sleep(rateLimitWaitMs);
+ attempt++;
+ continue;
+ }
+ if (response.status >= 500 && attempt < maxRetries) {
+ await sleep(1000 * 2 ** attempt);
+ attempt++;
+ continue;
+ }
+ if (response.status === 422 && errorBody.type === 'LIST_RECORDS_ITERATOR_NOT_AVAILABLE') {
+ throw new AirtableIteratorExpiredError();
+ }
+ throw new AirtableApiError(
+ errorBody.message || `Airtable API request failed with status ${response.status}`,
+ response.status,
+ errorBody.type
+ );
+ }
+ }
+
+ private async parseError(response: Response): Promise<{ type?: string; message?: string }> {
+ try {
+ const body = (await response.json()) as {
+ error?: string | { type?: string; message?: string };
+ };
+ if (typeof body.error === 'string') {
+ return { type: body.error };
+ }
+ return body.error ?? {};
+ } catch {
+ return {};
+ }
+ }
+}
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-formula-translator.spec.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-formula-translator.spec.ts
new file mode 100644
index 0000000000..05ac053296
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-formula-translator.spec.ts
@@ -0,0 +1,49 @@
+import { translateAirtableFormula } from './airtable-formula-translator';
+
+const expr = (formula: string): string => {
+ const result = translateAirtableFormula(formula);
+ if (!result.ok) throw new Error(`expected ok, got: ${result.reason}`);
+ return result.expression;
+};
+
+describe('translateAirtableFormula', () => {
+ it('preserves field references, operators, and literals verbatim', () => {
+ expect(expr('{fldabTo0bWtgfSrxf}+1')).toBe('{fldabTo0bWtgfSrxf}+1');
+ expect(expr('{fldA} & " - " & {fldB}')).toBe('{fldA} & " - " & {fldB}');
+ expect(expr('({fldA} + {fldB}) / 2 >= 10')).toBe('({fldA} + {fldB}) / 2 >= 10');
+ });
+
+ it('maps identical and renamed function names', () => {
+ expect(expr('CONCATENATE({fldA},"x")')).toBe('CONCATENATE({fldA},"x")');
+ expect(expr('ARRAYJOIN({fldA},", ")')).toBe('ARRAY_JOIN({fldA},", ")');
+ expect(expr('DATEADD({fldA},1,"days")')).toBe('DATE_ADD({fldA},1,"days")');
+ expect(expr('REGEX_REPLACE({fldA},"a","b")')).toBe('REGEXP_REPLACE({fldA},"a","b")');
+ expect(expr('ISERROR({fldA})')).toBe('IS_ERROR({fldA})');
+ });
+
+ it('resolves function names case-insensitively to the Teable canonical name', () => {
+ expect(expr('if({fldA}>5,1,2)')).toBe('IF({fldA}>5,1,2)');
+ });
+
+ it('converts TRUE()/FALSE() calls into boolean literals', () => {
+ expect(expr('IF({fldA},TRUE(),FALSE())')).toBe('IF({fldA},TRUE,FALSE)');
+ });
+
+ it('does not translate text that merely looks like a function inside a string', () => {
+ expect(expr('"ARRAYJOIN(x)"')).toBe('"ARRAYJOIN(x)"');
+ });
+
+ it('rejects unsupported functions so the importer can fall back to a snapshot', () => {
+ expect(translateAirtableFormula('REGEX_MATCH({fldA},"x")').ok).toBe(false);
+ expect(translateAirtableFormula('ARRAYSLICE({fldA},1,2)').ok).toBe(false);
+ });
+
+ it('rejects the unsupported "^" power operator', () => {
+ const result = translateAirtableFormula('{fldA}^2');
+ expect(result.ok).toBe(false);
+ });
+
+ it('rejects an unrecognized bare identifier', () => {
+ expect(translateAirtableFormula('FOO').ok).toBe(false);
+ });
+});
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-formula-translator.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-formula-translator.ts
new file mode 100644
index 0000000000..f9a34798d9
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-formula-translator.ts
@@ -0,0 +1,253 @@
+/**
+ * Translates an Airtable formula into a Teable formula expression.
+ *
+ * The two languages are close enough to translate faithfully for a large common
+ * subset: both reference fields as `{fldXXX}`, share the same operators
+ * (`+ - * / & = != < > <= >= && ||`) and string/number/boolean literal syntax,
+ * and overlap heavily on function names. Translation therefore only needs to:
+ * - remap function names that differ (e.g. ARRAYJOIN → ARRAY_JOIN),
+ * - turn Airtable's `TRUE()` / `FALSE()` calls into Teable's TRUE / FALSE
+ * boolean literals,
+ * - and refuse anything it cannot represent (an unknown function, the `^`
+ * power operator Teable's grammar lacks) so the importer can fall back to a
+ * static value snapshot instead of emitting a broken formula.
+ *
+ * Field references are preserved verbatim as `{fldXXX}` (Airtable field ids);
+ * the import pipeline remaps those ids to the created Teable field ids, exactly
+ * as it does for every other field-id reference.
+ */
+
+/** Airtable function name (upper-case) → Teable function name. */
+const airtableToTeableFunction: Record = {
+ // Numeric — identical names
+ ABS: 'ABS',
+ AVERAGE: 'AVERAGE',
+ CEILING: 'CEILING',
+ COUNT: 'COUNT',
+ COUNTA: 'COUNTA',
+ COUNTALL: 'COUNTALL',
+ EVEN: 'EVEN',
+ EXP: 'EXP',
+ FLOOR: 'FLOOR',
+ INT: 'INT',
+ LOG: 'LOG',
+ MAX: 'MAX',
+ MIN: 'MIN',
+ MOD: 'MOD',
+ ODD: 'ODD',
+ POWER: 'POWER',
+ ROUND: 'ROUND',
+ ROUNDDOWN: 'ROUNDDOWN',
+ ROUNDUP: 'ROUNDUP',
+ SQRT: 'SQRT',
+ SUM: 'SUM',
+ VALUE: 'VALUE',
+
+ // Text — identical names
+ CONCATENATE: 'CONCATENATE',
+ ENCODE_URL_COMPONENT: 'ENCODE_URL_COMPONENT',
+ FIND: 'FIND',
+ LEFT: 'LEFT',
+ LEN: 'LEN',
+ LOWER: 'LOWER',
+ MID: 'MID',
+ REPLACE: 'REPLACE',
+ REPT: 'REPT',
+ RIGHT: 'RIGHT',
+ SEARCH: 'SEARCH',
+ SUBSTITUTE: 'SUBSTITUTE',
+ T: 'T',
+ TRIM: 'TRIM',
+ UPPER: 'UPPER',
+ // Text — renamed
+ REGEX_REPLACE: 'REGEXP_REPLACE',
+
+ // Logical — identical names (TRUE/FALSE handled as literals, not here)
+ AND: 'AND',
+ BLANK: 'BLANK',
+ ERROR: 'ERROR',
+ IF: 'IF',
+ NOT: 'NOT',
+ OR: 'OR',
+ SWITCH: 'SWITCH',
+ XOR: 'XOR',
+ // Logical — renamed
+ ISERROR: 'IS_ERROR',
+
+ // Date/time — identical names
+ CREATED_TIME: 'CREATED_TIME',
+ DATESTR: 'DATESTR',
+ DATETIME_DIFF: 'DATETIME_DIFF',
+ DATETIME_FORMAT: 'DATETIME_FORMAT',
+ DATETIME_PARSE: 'DATETIME_PARSE',
+ DAY: 'DAY',
+ FROMNOW: 'FROMNOW',
+ HOUR: 'HOUR',
+ IS_AFTER: 'IS_AFTER',
+ IS_BEFORE: 'IS_BEFORE',
+ IS_SAME: 'IS_SAME',
+ LAST_MODIFIED_TIME: 'LAST_MODIFIED_TIME',
+ MINUTE: 'MINUTE',
+ MONTH: 'MONTH',
+ NOW: 'NOW',
+ SECOND: 'SECOND',
+ SET_LOCALE: 'SET_LOCALE',
+ SET_TIMEZONE: 'SET_TIMEZONE',
+ TIMESTR: 'TIMESTR',
+ TODAY: 'TODAY',
+ TONOW: 'TONOW',
+ WEEKDAY: 'WEEKDAY',
+ WEEKNUM: 'WEEKNUM',
+ WORKDAY: 'WORKDAY',
+ WORKDAY_DIFF: 'WORKDAY_DIFF',
+ YEAR: 'YEAR',
+ // Date/time — renamed
+ DATEADD: 'DATE_ADD',
+
+ // Array — renamed (Airtable drops the underscore)
+ ARRAYCOMPACT: 'ARRAY_COMPACT',
+ ARRAYFLATTEN: 'ARRAY_FLATTEN',
+ ARRAYJOIN: 'ARRAY_JOIN',
+ ARRAYUNIQUE: 'ARRAY_UNIQUE',
+
+ // Record
+ RECORD_ID: 'RECORD_ID',
+};
+
+export type IFormulaTranslation = { ok: true; expression: string } | { ok: false; reason: string };
+
+type ITokenType = 'string' | 'field' | 'number' | 'ident' | 'punct' | 'ws';
+interface IToken {
+ type: ITokenType;
+ raw: string;
+}
+
+// Airtable function names and the operators we recognize are all ASCII; field
+// names never reach the tokenizer (they are opaque `{...}` spans).
+const isWhitespace = (c: string) => c === ' ' || c === '\t' || c === '\r' || c === '\n';
+const isDigit = (c: string) => c >= '0' && c <= '9';
+const isIdentStart = (c: string) => /[a-z_]/i.test(c);
+const isIdentPart = (c: string) => /\w/.test(c);
+const twoCharOperators = new Set(['!=', '<=', '>=', '&&', '||']);
+
+/**
+ * Splits a formula into structural tokens, keeping string and `{field}` spans
+ * opaque so their contents are never mistaken for operators or identifiers.
+ * Returns null on an unterminated string or field reference.
+ */
+// eslint-disable-next-line sonarjs/cognitive-complexity -- a flat lexer dispatch
+const tokenize = (input: string): IToken[] | null => {
+ const tokens: IToken[] = [];
+ let i = 0;
+ while (i < input.length) {
+ const c = input[i];
+ if (c === '"' || c === "'") {
+ let j = i + 1;
+ while (j < input.length && input[j] !== c) {
+ if (input[j] === '\\') j += 1;
+ j += 1;
+ }
+ if (j >= input.length) return null;
+ tokens.push({ type: 'string', raw: input.slice(i, j + 1) });
+ i = j + 1;
+ } else if (c === '{') {
+ const end = input.indexOf('}', i + 1);
+ if (end < 0) return null;
+ tokens.push({ type: 'field', raw: input.slice(i, end + 1) });
+ i = end + 1;
+ } else if (isWhitespace(c)) {
+ let j = i + 1;
+ while (j < input.length && isWhitespace(input[j])) j += 1;
+ tokens.push({ type: 'ws', raw: input.slice(i, j) });
+ i = j;
+ } else if (isDigit(c) || (c === '.' && isDigit(input[i + 1] ?? ''))) {
+ let j = i + 1;
+ while (j < input.length && /[0-9.]/.test(input[j])) j += 1;
+ // optional exponent
+ if ((input[j] === 'e' || input[j] === 'E') && j < input.length) {
+ j += 1;
+ if (input[j] === '+' || input[j] === '-') j += 1;
+ while (j < input.length && isDigit(input[j])) j += 1;
+ }
+ tokens.push({ type: 'number', raw: input.slice(i, j) });
+ i = j;
+ } else if (isIdentStart(c)) {
+ let j = i + 1;
+ while (j < input.length && isIdentPart(input[j])) j += 1;
+ tokens.push({ type: 'ident', raw: input.slice(i, j) });
+ i = j;
+ } else {
+ const two = input.slice(i, i + 2);
+ if (twoCharOperators.has(two)) {
+ tokens.push({ type: 'punct', raw: two });
+ i += 2;
+ } else {
+ tokens.push({ type: 'punct', raw: c });
+ i += 1;
+ }
+ }
+ }
+ return tokens;
+};
+
+/** Index of the next non-whitespace token at or after `from`, or -1. */
+const nextSignificant = (tokens: IToken[], from: number): number => {
+ for (let k = from; k < tokens.length; k += 1) {
+ if (tokens[k].type !== 'ws') return k;
+ }
+ return -1;
+};
+
+// eslint-disable-next-line sonarjs/cognitive-complexity -- a flat token-rewrite loop
+export const translateAirtableFormula = (formula: string): IFormulaTranslation => {
+ const trimmed = formula.trim();
+ if (!trimmed) return { ok: false, reason: 'empty formula' };
+
+ const tokens = tokenize(formula);
+ if (!tokens)
+ return { ok: false, reason: 'unparseable formula (unterminated string or field reference)' };
+
+ const out: string[] = [];
+ for (let k = 0; k < tokens.length; k += 1) {
+ const token = tokens[k];
+ if (token.type === 'punct') {
+ if (token.raw === '^') {
+ return { ok: false, reason: 'unsupported operator "^" (power)' };
+ }
+ out.push(token.raw);
+ continue;
+ }
+ if (token.type !== 'ident') {
+ out.push(token.raw);
+ continue;
+ }
+
+ const upper = token.raw.toUpperCase();
+ const callParen = nextSignificant(tokens, k + 1);
+ const isCall = callParen >= 0 && tokens[callParen].raw === '(';
+
+ if (upper === 'TRUE' || upper === 'FALSE') {
+ out.push(upper);
+ if (isCall) {
+ // Drop Airtable's empty `()`: TRUE() / FALSE() → TRUE / FALSE.
+ const closeParen = nextSignificant(tokens, callParen + 1);
+ if (closeParen < 0 || tokens[closeParen].raw !== ')') {
+ return { ok: false, reason: `${upper}() called with arguments` };
+ }
+ k = closeParen;
+ }
+ continue;
+ }
+
+ if (!isCall) {
+ return { ok: false, reason: `unrecognized name "${token.raw}"` };
+ }
+ const mapped = airtableToTeableFunction[upper];
+ if (!mapped) {
+ return { ok: false, reason: `unsupported function "${token.raw}"` };
+ }
+ out.push(mapped);
+ }
+
+ return { ok: true, expression: out.join('') };
+};
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-import.controller.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-import.controller.ts
new file mode 100644
index 0000000000..e55e430b80
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-import.controller.ts
@@ -0,0 +1,118 @@
+import { BadRequestException, Body, Controller, Logger, Post, Res } from '@nestjs/common';
+import { type Action } from '@teable/core';
+import {
+ importAirtableAnalyzeRoSchema,
+ importAirtableRoSchema,
+ type IImportAirtableAnalyzeRo,
+ type IImportAirtableAnalyzeVo,
+ type IImportAirtableRo,
+} from '@teable/openapi';
+import { Response as ExpressResponse } from 'express';
+import { ClsService } from 'nestjs-cls';
+import type { IClsStore } from '../../types/cls';
+import { ZodValidationPipe } from '../../zod.validation.pipe';
+import { PermissionService } from '../auth/permission.service';
+import { AirtableApiError } from './airtable-api.client';
+import { AirtableImportService } from './airtable-import.service';
+
+const formatAirtableImportError = (error: unknown): string => {
+ if (error instanceof AirtableApiError) {
+ if (error.status === 401) {
+ return 'Airtable rejected the access token. Check that the token is valid.';
+ }
+ if (error.status === 403 || error.status === 404) {
+ return 'Airtable base not accessible. Grant the token access to the base and the "data.records:read" and "schema.bases:read" scopes.';
+ }
+ return `Airtable API error: ${error.message}`;
+ }
+ return error instanceof Error && error.message ? error.message : 'Unknown import error';
+};
+
+@Controller('api/base')
+export class AirtableImportController {
+ private readonly logger = new Logger(AirtableImportController.name);
+
+ constructor(
+ private readonly airtableImportService: AirtableImportService,
+ private readonly permissionService: PermissionService,
+ private readonly cls: ClsService
+ ) {}
+
+ @Post('import-airtable/analyze')
+ async analyze(
+ @Body(new ZodValidationPipe(importAirtableAnalyzeRoSchema))
+ analyzeRo: IImportAirtableAnalyzeRo
+ ): Promise {
+ try {
+ return await this.airtableImportService.analyze(analyzeRo);
+ } catch (error) {
+ if (error instanceof AirtableApiError) {
+ throw new BadRequestException(formatAirtableImportError(error));
+ }
+ throw error;
+ }
+ }
+
+ @Post('import-airtable/stream')
+ async importStream(
+ @Body(new ZodValidationPipe(importAirtableRoSchema))
+ importAirtableRo: IImportAirtableRo,
+ @Res() res: ExpressResponse
+ ) {
+ // Authorize the real write target before doing anything: importing into an
+ // existing base needs table-import rights on THAT base — not base|create on
+ // whatever space the caller passes — while creating a new base needs
+ // base|create on the target space. (validPermissions intersects token scopes.)
+ const targetResourceId = importAirtableRo.baseId ?? importAirtableRo.spaceId;
+ if (!targetResourceId) {
+ // Unreachable via the zod schema (spaceId is required when baseId is absent),
+ // but keep the guard so the permission target is always a concrete resource.
+ throw new BadRequestException('Either baseId or spaceId is required.');
+ }
+ const requiredPermissions: Action[] = importAirtableRo.baseId
+ ? ['base|table_import']
+ : ['base|create'];
+ await this.permissionService.validPermissions(
+ targetResourceId,
+ requiredPermissions,
+ this.cls.get('accessTokenId')
+ );
+
+ const sseHeartbeatMs = 15_000;
+ res.setHeader('Content-Type', 'text/event-stream');
+ res.setHeader('Cache-Control', 'no-cache, no-transform');
+ res.setHeader('Connection', 'keep-alive');
+ res.setHeader('X-Accel-Buffering', 'no');
+ res.flushHeaders();
+
+ const isStreamClosed = () => res.writableEnded || res.destroyed;
+ const sendEvent = (data: unknown) => {
+ if (isStreamClosed()) return;
+ res.write(`data: ${JSON.stringify(data)}\n\n`);
+ (res as ExpressResponse & { flush?: () => void }).flush?.();
+ };
+ const heartbeat = setInterval(() => {
+ if (isStreamClosed()) return;
+ res.write(': ping\n\n');
+ (res as ExpressResponse & { flush?: () => void }).flush?.();
+ }, sseHeartbeatMs);
+ res.on('close', () => clearInterval(heartbeat));
+
+ try {
+ const result = await this.airtableImportService.importBase(importAirtableRo, (progress) => {
+ sendEvent({ type: 'progress', ...progress });
+ });
+ sendEvent({ type: 'done', data: result });
+ } catch (error) {
+ const reason = formatAirtableImportError(error);
+ this.logger.warn(
+ `[airtable-import] failed airtableBase=${importAirtableRo.airtableBaseId} ` +
+ `target=${importAirtableRo.baseId ?? 'new'} reason=${reason}`
+ );
+ sendEvent({ type: 'error', message: reason });
+ } finally {
+ clearInterval(heartbeat);
+ res.end();
+ }
+ }
+}
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-import.module.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-import.module.ts
new file mode 100644
index 0000000000..3cb03287ea
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-import.module.ts
@@ -0,0 +1,30 @@
+import { Module } from '@nestjs/common';
+import { AiModule } from '../ai/ai.module';
+import { AttachmentsModule } from '../attachments/attachments.module';
+import { StorageModule } from '../attachments/plugins/storage.module';
+import { PermissionModule } from '../auth/permission.module';
+import { BaseModule } from '../base/base.module';
+import { FieldOpenApiModule } from '../field/open-api/field-open-api.module';
+import { RecordOpenApiModule } from '../record/open-api/record-open-api.module';
+import { TableOpenApiModule } from '../table/open-api/table-open-api.module';
+import { ViewOpenApiModule } from '../view/open-api/view-open-api.module';
+import { AirtableImportController } from './airtable-import.controller';
+import { AirtableImportService } from './airtable-import.service';
+
+@Module({
+ imports: [
+ BaseModule,
+ TableOpenApiModule,
+ RecordOpenApiModule,
+ FieldOpenApiModule,
+ ViewOpenApiModule,
+ AttachmentsModule,
+ StorageModule,
+ AiModule,
+ PermissionModule,
+ ],
+ controllers: [AirtableImportController],
+ providers: [AirtableImportService],
+ exports: [AirtableImportService],
+})
+export class AirtableImportModule {}
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.spec.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.spec.ts
new file mode 100644
index 0000000000..14e73f548e
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.spec.ts
@@ -0,0 +1,88 @@
+import { FieldType, Relationship } from '@teable/core';
+import type { IImportAirtableIssue } from '@teable/openapi';
+import { describe, expect, it } from 'vitest';
+import { AirtableImportService } from './airtable-import.service';
+import type { IPlannedDirectField, IPlannedLinkField } from './airtable-schema-mapper';
+
+const service = new AirtableImportService(
+ null as never,
+ null as never,
+ null as never,
+ null as never,
+ null as never,
+ null as never,
+ null as never,
+ null as never,
+ null as never,
+ undefined
+);
+
+const linkField = (prefersSingle: boolean): IPlannedLinkField => ({
+ airtableFieldId: 'fldLink',
+ teableFieldId: 'fldTeable0000000001',
+ name: 'Link',
+ airtableForeignTableId: 'tblForeign',
+ prefersSingle,
+});
+
+describe('AirtableImportService.decideRelationship', () => {
+ it('follows the declared Airtable relationship', () => {
+ expect(service['decideRelationship'](linkField(true))).toBe(Relationship.ManyOne);
+ expect(service['decideRelationship'](linkField(false))).toBe(Relationship.ManyMany);
+ });
+});
+
+describe('AirtableImportService.applyAiConfig', () => {
+ const plannedAiField = (): IPlannedDirectField => ({
+ airtableFieldId: 'fldAi',
+ converter: 'aiText',
+ aiPromptParts: [
+ { text: 'Summarize ' },
+ { airtableFieldId: 'fldTitle', fieldName: 'Title' },
+ { airtableFieldId: 'fldGone', fieldName: 'Gone' },
+ ],
+ ro: { id: 'fldTeableAi00000001', name: 'Summary', type: FieldType.LongText, options: {} },
+ });
+ const fieldIdMap = { fldTitle: 'fldTeableTitle00001' };
+
+ it('builds a customization AI config with mapped field references', () => {
+ const issues: IImportAirtableIssue[] = [];
+ const ro = service['applyAiConfig'](
+ plannedAiField(),
+ 'openai@gpt@main',
+ fieldIdMap,
+ 'T',
+ issues
+ );
+ expect(ro.aiConfig).toMatchObject({
+ type: 'customization',
+ modelKey: 'openai@gpt@main',
+ prompt: 'Summarize {fldTeableTitle00001}Gone',
+ isAutoFill: false,
+ });
+ expect(issues).toHaveLength(0);
+ });
+
+ it('keeps the snapshot and reports when no AI model is configured', () => {
+ const issues: IImportAirtableIssue[] = [];
+ const ro = service['applyAiConfig'](plannedAiField(), undefined, fieldIdMap, 'T', issues);
+ expect(ro.aiConfig).toBeUndefined();
+ expect(issues[0]).toMatchObject({
+ code: 'fieldDegraded',
+ fieldName: 'Summary',
+ fromType: 'aiText',
+ reason: 'no AI model is configured',
+ });
+ });
+
+ it('passes non-AI fields through untouched', () => {
+ const issues: IImportAirtableIssue[] = [];
+ const planned: IPlannedDirectField = {
+ airtableFieldId: 'fldText',
+ converter: 'string',
+ ro: { name: 'Plain', type: FieldType.SingleLineText, options: {} },
+ };
+ expect(service['applyAiConfig'](planned, 'model@x@y', {}, 'T', issues)).toBe(planned.ro);
+ expect(issues).toHaveLength(0);
+ });
+});
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.ts
new file mode 100644
index 0000000000..8b55e0cd14
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-import.service.ts
@@ -0,0 +1,1628 @@
+import { Readable } from 'node:stream';
+import { BadRequestException, Inject, Injectable, Logger, Optional } from '@nestjs/common';
+import type {
+ IColumnMetaRo,
+ IFieldRo,
+ IFilter,
+ ILinkFieldOptions,
+ IUserFieldOptions,
+ IViewOptions,
+ ViewType,
+ CellValueType,
+} from '@teable/core';
+import { FieldAIActionType, FieldKeyType, FieldType, Relationship } from '@teable/core';
+import { PrismaService } from '@teable/db-main-prisma';
+import type {
+ IImportAirtableAnalyzeRo,
+ IImportAirtableAnalyzeVo,
+ IImportAirtableIssue,
+ IImportAirtableRo,
+ IImportAirtableVo,
+} from '@teable/openapi';
+import { CollaboratorType, PrincipalType, UploadType } from '@teable/openapi';
+import { AiService } from '../ai/ai.service';
+import { AttachmentsService } from '../attachments/attachments.service';
+import StorageAdapter from '../attachments/plugins/adapter';
+import { InjectStorageAdapter } from '../attachments/plugins/storage';
+import { BaseService } from '../base/base.service';
+import { FieldOpenApiV2Service } from '../field/open-api/field-open-api-v2.service';
+import { RecordOpenApiV2Service } from '../record/open-api/record-open-api-v2.service';
+import { TableOpenApiV2Service } from '../table/open-api/table-open-api-v2.service';
+import { ViewOpenApiService } from '../view/open-api/view-open-api.service';
+import { AirtableApiClient, AirtableIteratorExpiredError } from './airtable-api.client';
+import { AirtableLinkRowSpill, type ISpilledLinkRow } from './airtable-link-spill';
+import {
+ convertAirtableCellValue,
+ convertCollaboratorCellValue,
+ extractLinkedRecordIds,
+ type IResolvedSpaceUser,
+} from './airtable-record-converter';
+import type {
+ IAirtableImportPlan,
+ IAirtableTablePlan,
+ IPlannedDirectField,
+ IPlannedFormulaField,
+ IPlannedLinkField,
+} from './airtable-schema-mapper';
+import { buildAirtableImportPlan } from './airtable-schema-mapper';
+import { AirtableShareClient, AirtableShareError } from './airtable-share.client';
+import {
+ AIRTABLE_IMPORT_TOKEN_RESOLVER,
+ type IAirtableImportTokenResolver,
+} from './airtable-token-resolver';
+import {
+ mapAirtableFilter,
+ mapAirtableViewConfig,
+ type IImportFieldMeta,
+ type IViewConfigMapperContext,
+} from './airtable-view-config-mapper';
+import type { IAirtableAttachment, IAirtableRecord, IAirtableTable } from './airtable.types';
+
+export interface IAirtableImportProgress {
+ phase: string;
+ detail?: string;
+ tableName?: string;
+ tableIndex?: number;
+ totalTables?: number;
+ processedRows?: number;
+}
+
+export type IAirtableImportProgressReporter = (progress: IAirtableImportProgress) => void;
+
+const linkUpdateBatchSize = 100;
+const attachmentConcurrency = 3;
+const maxListRestarts = 2;
+const unknownErrorText = 'unknown error';
+
+interface ILinkFieldRuntime {
+ plan: IPlannedLinkField;
+ tableAirtableId: string;
+ relationship: Relationship;
+}
+
+/** A created Teable view paired with the Airtable view its config comes from. */
+interface IViewConfigTarget {
+ airtableViewId: string;
+ teableViewId: string;
+ teableViewType: ViewType;
+ tableId: string;
+ tableName: string;
+ viewName: string;
+}
+
+/** Runs tasks with bounded concurrency, preserving the result order. */
+const mapWithConcurrency = async (
+ items: T[],
+ limit: number,
+ task: (item: T) => Promise
+): Promise => {
+ const results: R[] = new Array(items.length);
+ let next = 0;
+ const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ const index = next++;
+ if (index >= items.length) return;
+ results[index] = await task(items[index]);
+ }
+ });
+ await Promise.all(workers);
+ return results;
+};
+
+@Injectable()
+export class AirtableImportService {
+ private readonly logger = new Logger(AirtableImportService.name);
+
+ constructor(
+ private readonly baseService: BaseService,
+ private readonly tableOpenApiV2Service: TableOpenApiV2Service,
+ private readonly recordOpenApiV2Service: RecordOpenApiV2Service,
+ private readonly fieldOpenApiV2Service: FieldOpenApiV2Service,
+ private readonly viewOpenApiService: ViewOpenApiService,
+ private readonly attachmentsService: AttachmentsService,
+ private readonly prismaService: PrismaService,
+ private readonly aiService: AiService,
+ @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter,
+ @Optional()
+ @Inject(AIRTABLE_IMPORT_TOKEN_RESOLVER)
+ private readonly tokenResolver?: IAirtableImportTokenResolver
+ ) {}
+
+ /**
+ * Stages link-cell spill parts in the deployment's blob storage
+ * (local/S3/MinIO), mirroring how the .tea import streams its data files —
+ * no container-local temp files.
+ */
+ private createLinkSpill(): AirtableLinkRowSpill {
+ const bucket = StorageAdapter.getBucket(UploadType.Import);
+ return new AirtableLinkRowSpill({
+ upload: async (path, data) => {
+ await this.storageAdapter.uploadFileStream(bucket, path, data);
+ },
+ download: async (path) => {
+ const stream = await this.storageAdapter.downloadFile(bucket, path);
+ return stream instanceof Readable
+ ? stream
+ : // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ Readable.fromWeb(stream as any);
+ },
+ cleanup: async (dir) => {
+ await this.storageAdapter.deleteDir(bucket, dir);
+ },
+ });
+ }
+
+ /**
+ * Builds the per-request access token provider. With an integrationId the
+ * token stays server-side and is refreshed by the resolver; a raw
+ * accessToken (direct API usage) is used as-is.
+ */
+ private createClient(ro: { integrationId?: string; accessToken?: string }): AirtableApiClient {
+ if (ro.integrationId) {
+ const resolver = this.tokenResolver;
+ if (!resolver) {
+ throw new BadRequestException('Airtable integrations are not available on this instance');
+ }
+ const integrationId = ro.integrationId;
+ return new AirtableApiClient(() => resolver.resolveAccessToken(integrationId));
+ }
+ if (!ro.accessToken) {
+ throw new BadRequestException('Either integrationId or accessToken is required');
+ }
+ const accessToken = ro.accessToken;
+ return new AirtableApiClient(() => accessToken);
+ }
+
+ async analyze(ro: IImportAirtableAnalyzeRo): Promise {
+ const client = this.createClient(ro);
+ if (!ro.airtableBaseId) {
+ const bases = await client.listBases();
+ return {
+ bases: bases
+ .filter((base) => base.permissionLevel !== 'none')
+ .map(({ id, name, permissionLevel }) => ({ id, name, permissionLevel })),
+ };
+ }
+
+ const tables = await client.getBaseSchema(ro.airtableBaseId);
+ const plan = buildAirtableImportPlan(tables);
+ return {
+ base: {
+ id: ro.airtableBaseId,
+ tables: tables.map((table) => ({
+ id: table.id,
+ name: table.name,
+ fieldCount: table.fields.length,
+ viewCount: table.views.length,
+ })),
+ issues: plan.issues,
+ },
+ };
+ }
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity -- orchestrates the linear import pipeline
+ async importBase(
+ ro: IImportAirtableRo,
+ onProgress?: IAirtableImportProgressReporter
+ ): Promise {
+ const importRecords = ro.importRecords ?? true;
+ const importAttachments = ro.importAttachments ?? true;
+ const importViewConfig = ro.importViewConfig === true;
+ const progress = (event: IAirtableImportProgress) => onProgress?.(event);
+ const client = this.createClient(ro);
+ const startedAt = Date.now();
+
+ progress({ phase: 'fetching_schema' });
+ const airtableTables = await client.getBaseSchema(ro.airtableBaseId);
+ this.logger.log(
+ `[airtable-import] start airtableBase=${ro.airtableBaseId} target=${ro.baseId ?? 'new'} ` +
+ `tables=${airtableTables.length} source=${ro.integrationId ? 'integration' : 'pat'} ` +
+ `records=${importRecords} attachments=${importAttachments} viewConfig=${importViewConfig}`
+ );
+
+ // Resolve and validate the shared-base link up front (fail fast on a wrong or
+ // private link) and read the base model it unlocks, which carries the rollup
+ // aggregations the official API never exposes — so rollups can be recreated
+ // live instead of snapshotted.
+ const shareClient = importViewConfig ? await this.resolveShareClient(ro) : undefined;
+ const rollupSources = shareClient ? await shareClient.fetchApplicationModel() : undefined;
+
+ const plan = buildAirtableImportPlan(airtableTables, rollupSources);
+ const issues: IImportAirtableIssue[] = [...plan.issues];
+ const totalTables = plan.tables.length;
+
+ // Import into an existing base (adding its tables) or create a fresh one.
+ // spaceId is only needed to create a new base; for an existing base the
+ // base's own space is used (see user-field resolution below).
+ let base;
+ if (ro.baseId) {
+ base = await this.baseService.getBaseById(ro.baseId);
+ } else {
+ if (!ro.spaceId) {
+ throw new BadRequestException('spaceId is required when baseId is not provided.');
+ }
+ progress({ phase: 'creating_base', detail: ro.baseName });
+ base = await this.baseService.createBase({
+ spaceId: ro.spaceId,
+ name: ro.baseName ?? 'Imported base',
+ });
+ }
+ const aiModelKey = await this.resolveAiModelKey(base.id);
+
+ // airtable table id -> created teable table id
+ const tableIdMap: Record = {};
+ const tableViewIds: Record = {};
+ // airtable view id -> created teable view id; used for a link's
+ // "limit record selection to a view", regardless of importViewConfig.
+ const viewIdMap: Record = {};
+ const viewTargets: IViewConfigTarget[] = [];
+ for (const [index, tablePlan] of plan.tables.entries()) {
+ progress({
+ phase: 'creating_table',
+ tableName: tablePlan.name,
+ tableIndex: index + 1,
+ totalTables,
+ });
+ const table = await this.tableOpenApiV2Service.createTable(base.id, {
+ name: tablePlan.name,
+ description: tablePlan.description,
+ fieldKeyType: FieldKeyType.Id,
+ fields: tablePlan.fields.map((field) =>
+ this.applyAiConfig(field, aiModelKey, plan.fieldIdMap, tablePlan.name, issues)
+ ),
+ views: tablePlan.views,
+ records: [],
+ });
+ tableIdMap[tablePlan.airtableTableId] = table.id;
+ tableViewIds[table.id] = (table.views ?? []).map((view) => view.id);
+ tablePlan.viewSources.forEach((source, viewIndex) => {
+ const createdView = (table.views ?? [])[viewIndex];
+ if (createdView?.type === source.teableViewType) {
+ viewIdMap[source.airtableViewId] = createdView.id;
+ }
+ });
+ if (importViewConfig) {
+ this.collectViewTargets(tablePlan, table.id, table.views ?? [], viewTargets);
+ }
+ }
+
+ // Complete the table structure before data with link fields only — they
+ // follow the relationship Airtable declares. Derived fields (lookups,
+ // counts) and view configuration are applied AFTER the records, so a field
+ // that cannot be computed degrades to a reported issue instead of breaking
+ // record/link writes and aborting the whole import.
+ progress({ phase: 'creating_links' });
+ const linkRuntimes = await this.createLinkFields({ plan, tableIdMap, viewIdMap, issues });
+
+ // Memory bounds for large bases: record pages, inserts, attachment
+ // transfers and link writes are all streamed/batched. The only data kept
+ // for the whole run is the old->new record id map of tables that links
+ // point at (needed to remap link cells); buffered link rows spill to the
+ // blob storage.
+ const linkTargetTables = new Set(
+ plan.tables.flatMap((table) => table.linkFields.map((link) => link.airtableForeignTableId))
+ );
+ const recordIdMaps = new Map>();
+ // Single-link fields whose data turned out to hold several links — relaxed
+ // to many-to-many before fill so no link is dropped (Airtable's single-link
+ // is a soft per-cell preference, not an enforced 1:1).
+ const linkFieldsWithMulti = new Set();
+ const linkSpill = this.createLinkSpill();
+
+ try {
+ if (importRecords) {
+ // Resolve user fields against the base's actual space, not ro.spaceId —
+ // for an existing-base import the two can differ (ro.spaceId is optional
+ // there), and the base's members are the correct mapping target.
+ const usersByEmail = await this.getSpaceUsersByEmail(base.spaceId);
+ for (const [index, tablePlan] of plan.tables.entries()) {
+ await this.importTableRecords({
+ client,
+ ro,
+ tablePlan,
+ tableIndex: index + 1,
+ totalTables,
+ tableId: tableIdMap[tablePlan.airtableTableId],
+ usersByEmail,
+ importAttachments,
+ recordIdMaps,
+ linkSpill,
+ linkFieldsWithMulti,
+ issues,
+ progress,
+ });
+ if (!linkTargetTables.has(tablePlan.airtableTableId)) {
+ // The map was only needed for in-table restart deduplication.
+ recordIdMaps.delete(tablePlan.airtableTableId);
+ }
+ }
+
+ await this.relaxOversizedSingleLinks({
+ plan,
+ linkRuntimes,
+ linkFieldsWithMulti,
+ tableIdMap,
+ issues,
+ });
+
+ await this.fillLinkValues({
+ plan,
+ tableIdMap,
+ linkRuntimes,
+ recordIdMaps,
+ linkSpill,
+ issues,
+ progress,
+ });
+ }
+ } finally {
+ await linkSpill.cleanup();
+ }
+
+ // Derived fields are computed over the imported data; create them last so a
+ // single uncomputable field is reported, never fatal to the whole import.
+ await this.createLookupFields(plan, tableIdMap, issues);
+ await this.createCountFields(plan, tableIdMap, issues);
+ await this.createRollupFields(plan, tableIdMap, airtableTables, issues);
+ await this.createFormulaFields(plan, tableIdMap, issues);
+
+ // Restore Airtable's field order: links and derived fields are created in
+ // later phases and would otherwise sit at the end. Rewrite each view's
+ // column order to the source table's field order so the layout matches.
+ await this.reorderFields({
+ airtableTables,
+ tableIdMap,
+ tableViewIds,
+ fieldIdMap: plan.fieldIdMap,
+ });
+
+ // View configuration last: every field now exists, so sorts/groups that
+ // reference a lookup or count resolve too.
+ if (shareClient) {
+ await this.applyViewConfigs({
+ shareClient,
+ airtableTables,
+ plan,
+ viewTargets,
+ issues,
+ progress,
+ });
+ }
+
+ this.logImportOutcome(ro, base.id, tableIdMap, plan.fieldIdMap, issues, startedAt);
+ progress({ phase: 'import_done', detail: base.name });
+ return {
+ base,
+ tableIdMap,
+ fieldIdMap: plan.fieldIdMap,
+ issues,
+ };
+ }
+
+ /**
+ * Emits structured, collectable logs of an import's outcome: one summary line
+ * (counts + issue breakdown + duration) and one line per issue (which field /
+ * view degraded or was skipped, and why). Skips are `warn`, degrades are `log`,
+ * so a log pipeline can alert on the former and aggregate both by type/reason.
+ */
+ /**
+ * Restores the source field order. Plain fields are created in order with the
+ * table, but links and derived fields (lookup/count/rollup/formula) are added
+ * in later phases and land at the end; this rewrites every view's column order
+ * to the Airtable table's field order. Best-effort — a failure is logged, not
+ * fatal, since the data is already imported.
+ */
+ private async reorderFields(params: {
+ airtableTables: IAirtableTable[];
+ tableIdMap: Record;
+ tableViewIds: Record;
+ fieldIdMap: Record;
+ }): Promise {
+ const { airtableTables, tableIdMap, tableViewIds, fieldIdMap } = params;
+ for (const airtableTable of airtableTables) {
+ const tableId = tableIdMap[airtableTable.id];
+ const viewIds = tableViewIds[tableId] ?? [];
+ if (!tableId || viewIds.length === 0) continue;
+ const columnMetaRo = airtableTable.fields
+ .map((field) => fieldIdMap[field.id])
+ .filter((id): id is string => Boolean(id))
+ .map((fieldId, order) => ({ fieldId, columnMeta: { order } }));
+ if (columnMetaRo.length === 0) continue;
+ for (const viewId of viewIds) {
+ try {
+ await this.viewOpenApiService.updateViewColumnMeta(
+ tableId,
+ viewId,
+ columnMetaRo as IColumnMetaRo
+ );
+ } catch (error) {
+ this.logger.warn(
+ `[airtable-import] reorder failed for ${airtableTable.name}: ${
+ error instanceof Error ? error.message : 'error'
+ }`
+ );
+ }
+ }
+ }
+ }
+
+ private logImportOutcome(
+ ro: IImportAirtableRo,
+ baseId: string,
+ tableIdMap: Record,
+ fieldIdMap: Record,
+ issues: IImportAirtableIssue[],
+ startedAt: number
+ ): void {
+ const byCode = issues.reduce>((acc, issue) => {
+ acc[issue.code] = (acc[issue.code] ?? 0) + 1;
+ return acc;
+ }, {});
+ this.logger.log(
+ `[airtable-import] done base=${baseId} airtableBase=${ro.airtableBaseId} ` +
+ `tables=${Object.keys(tableIdMap).length} fields=${Object.keys(fieldIdMap).length} ` +
+ `issues=${issues.length} byCode=${JSON.stringify(byCode)} durationMs=${Date.now() - startedAt}`
+ );
+ for (const issue of issues) {
+ const target = `${issue.tableName}/${issue.fieldName ?? issue.viewName ?? '?'}`;
+ const change = issue.toType ? ` -> ${issue.toType}` : issue.reason ? `: ${issue.reason}` : '';
+ const line = `[airtable-import] ${issue.code} base=${baseId} ${target} (from=${issue.fromType ?? '?'}${change})`;
+ if (issue.code === 'fieldSkipped' || issue.code === 'viewSkipped') {
+ this.logger.warn(line);
+ } else {
+ this.logger.log(line);
+ }
+ }
+ }
+
+ /**
+ * Resolves the AI model used for imported Airtable aiText fields from the
+ * base's AI configuration (space integration or instance settings). Returns
+ * undefined when no AI model is configured.
+ */
+ private async resolveAiModelKey(baseId: string): Promise {
+ try {
+ const config = await this.aiService.getAIConfig(baseId);
+ return config.chatModel?.lg || undefined;
+ } catch {
+ return undefined;
+ }
+ }
+
+ /**
+ * Turns an Airtable aiText field into a Teable AI field (custom prompt with
+ * field references) when an AI model is available; otherwise keeps the
+ * plain long-text snapshot and reports the degradation.
+ */
+ private applyAiConfig(
+ planned: IPlannedDirectField,
+ aiModelKey: string | undefined,
+ fieldIdMap: Record,
+ tableName: string,
+ issues: IImportAirtableIssue[]
+ ): IFieldRo {
+ if (!planned.aiPromptParts) {
+ return planned.ro;
+ }
+ const prompt = planned.aiPromptParts
+ .map((part) => {
+ if (part.text != null) return part.text;
+ const teableFieldId = part.airtableFieldId && fieldIdMap[part.airtableFieldId];
+ if (teableFieldId) return `{${teableFieldId}}`;
+ return part.fieldName ?? '';
+ })
+ .join('');
+
+ if (!aiModelKey || !prompt.trim()) {
+ issues.push({
+ code: 'fieldDegraded',
+ tableName,
+ fieldName: planned.ro.name as string,
+ fromType: 'aiText',
+ toType: 'longText snapshot',
+ reason: aiModelKey ? 'the AI prompt is empty' : 'no AI model is configured',
+ });
+ return planned.ro;
+ }
+
+ return {
+ ...planned.ro,
+ aiConfig: {
+ type: FieldAIActionType.Customization,
+ modelKey: aiModelKey,
+ prompt,
+ // Keep imported snapshot values; users can enable auto-fill later to
+ // avoid triggering a generation for every imported record.
+ isAutoFill: false,
+ },
+ } as IFieldRo;
+ }
+
+ private async resolveShareClient(ro: IImportAirtableRo): Promise {
+ const shareClient = new AirtableShareClient();
+ try {
+ await shareClient.resolveShare(ro.shareLink ?? '');
+ shareClient.assertBaseMatch(ro.airtableBaseId);
+ } catch (error) {
+ if (error instanceof AirtableShareError) {
+ throw new BadRequestException(error.message);
+ }
+ throw error;
+ }
+ return shareClient;
+ }
+
+ /**
+ * Pairs each created Teable view with the Airtable view it came from. The v2
+ * create-table mapper preserves view order, so the i-th created view matches
+ * the i-th planned source; the type guard skips any unexpected drift.
+ */
+ private collectViewTargets(
+ tablePlan: IAirtableTablePlan,
+ tableId: string,
+ createdViews: Array<{ id: string; type: ViewType; name: string }>,
+ targets: IViewConfigTarget[]
+ ) {
+ tablePlan.viewSources.forEach((source, index) => {
+ const created = createdViews[index];
+ if (!created || created.type !== source.teableViewType) return;
+ targets.push({
+ airtableViewId: source.airtableViewId,
+ teableViewId: created.id,
+ teableViewType: source.teableViewType,
+ tableId,
+ tableName: tablePlan.name,
+ viewName: (tablePlan.views[index]?.name as string) ?? created.name,
+ });
+ });
+ }
+
+ /**
+ * Reads each view's configuration from the shared base and applies the mapped
+ * filter/sort/grouping/options. Every view and every aspect is isolated so a
+ * single failure degrades to an issue without affecting the rest.
+ */
+ private async applyViewConfigs(params: {
+ shareClient: AirtableShareClient;
+ airtableTables: IAirtableTable[];
+ plan: IAirtableImportPlan;
+ viewTargets: IViewConfigTarget[];
+ issues: IImportAirtableIssue[];
+ progress: (event: IAirtableImportProgress) => void;
+ }) {
+ const { shareClient, airtableTables, plan, viewTargets, issues, progress } = params;
+ if (viewTargets.length === 0) return;
+ progress({ phase: 'applying_view_config' });
+
+ const optionNames = this.buildSelectOptionNames(airtableTables);
+ const fieldMetaByTable = new Map>();
+
+ for (const target of viewTargets) {
+ try {
+ const config = await shareClient.fetchViewConfig(target.airtableViewId);
+ let metaMap = fieldMetaByTable.get(target.tableId);
+ if (!metaMap) {
+ metaMap = await this.fetchFieldMeta(target.tableId);
+ fieldMetaByTable.set(target.tableId, metaMap);
+ }
+ const ctx: IViewConfigMapperContext = {
+ resolveField: (columnId) => {
+ const teableFieldId = plan.fieldIdMap[columnId];
+ return teableFieldId ? metaMap?.get(teableFieldId) : undefined;
+ },
+ resolveSelectOptionName: (columnId, optionId) => optionNames.get(columnId)?.get(optionId),
+ };
+ const mapped = mapAirtableViewConfig({
+ teableViewType: target.teableViewType,
+ config,
+ ctx,
+ tableName: target.tableName,
+ viewName: target.viewName,
+ issues,
+ });
+ await this.applyMappedViewConfig(target, mapped, issues);
+ } catch (error) {
+ this.logger.warn(
+ `Failed to import view config for "${target.viewName}": ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`
+ );
+ issues.push({
+ code: 'viewConfigDegraded',
+ tableName: target.tableName,
+ viewName: target.viewName,
+ reason: 'could not read the view configuration from the shared base',
+ });
+ }
+ }
+ }
+
+ private async applyMappedViewConfig(
+ target: IViewConfigTarget,
+ mapped: ReturnType,
+ issues: IImportAirtableIssue[]
+ ) {
+ const apply = async (label: string, run: () => Promise) => {
+ try {
+ await run();
+ } catch (error) {
+ this.logger.warn(
+ `Failed to apply ${label} to view "${target.viewName}": ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`
+ );
+ issues.push({
+ code: 'viewConfigDegraded',
+ tableName: target.tableName,
+ viewName: target.viewName,
+ reason: `could not apply ${label}`,
+ });
+ }
+ };
+
+ if (mapped.filter) {
+ await apply('filters', () =>
+ this.viewOpenApiService.setViewProperty(
+ target.tableId,
+ target.teableViewId,
+ 'filter',
+ mapped.filter
+ )
+ );
+ }
+ if (mapped.sort) {
+ await apply('sorting', () =>
+ this.viewOpenApiService.setViewProperty(
+ target.tableId,
+ target.teableViewId,
+ 'sort',
+ mapped.sort
+ )
+ );
+ }
+ if (mapped.group) {
+ await apply('grouping', () =>
+ this.viewOpenApiService.setViewProperty(
+ target.tableId,
+ target.teableViewId,
+ 'group',
+ mapped.group
+ )
+ );
+ }
+ if (mapped.options) {
+ await apply('view options', () =>
+ this.viewOpenApiService.patchViewOptions(
+ target.tableId,
+ target.teableViewId,
+ mapped.options as IViewOptions
+ )
+ );
+ }
+ }
+
+ /** airtable select field id -> (option id -> option name) for filter mapping. */
+ private buildSelectOptionNames(tables: IAirtableTable[]): Map> {
+ const map = new Map>();
+ const selectFields = tables.flatMap((table) =>
+ table.fields.filter(
+ (field) => field.type === 'singleSelect' || field.type === 'multipleSelects'
+ )
+ );
+ for (const field of selectFields) {
+ const choices = field.options?.choices;
+ if (!Array.isArray(choices)) continue;
+ const byId = new Map();
+ for (const choice of choices) {
+ if (choice?.id && typeof choice.name === 'string') byId.set(choice.id, choice.name);
+ }
+ map.set(field.id, byId);
+ }
+ return map;
+ }
+
+ private async fetchFieldMeta(tableId: string): Promise> {
+ const fields = await this.prismaService.field.findMany({
+ where: { tableId, deletedTime: null },
+ select: { id: true, type: true, cellValueType: true, isMultipleCellValue: true },
+ });
+ return new Map(
+ fields.map((field) => [
+ field.id,
+ {
+ fieldId: field.id,
+ type: field.type as FieldType,
+ cellValueType: field.cellValueType as CellValueType,
+ isMultipleCellValue: field.isMultipleCellValue === true,
+ },
+ ])
+ );
+ }
+
+ private async getSpaceUsersByEmail(spaceId: string): Promise> {
+ const collaborators = await this.prismaService.collaborator.findMany({
+ where: {
+ resourceId: spaceId,
+ resourceType: CollaboratorType.Space,
+ principalType: PrincipalType.User,
+ },
+ select: { principalId: true },
+ });
+ if (collaborators.length === 0) {
+ return new Map();
+ }
+ const users = await this.prismaService.user.findMany({
+ where: {
+ id: { in: collaborators.map(({ principalId }) => principalId) },
+ deletedTime: null,
+ },
+ select: { id: true, name: true, email: true },
+ });
+ return new Map(
+ users.map((user) => [
+ user.email.toLowerCase(),
+ { id: user.id, name: user.name, email: user.email },
+ ])
+ );
+ }
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+ private async importTableRecords(params: {
+ client: AirtableApiClient;
+ ro: IImportAirtableRo;
+ tablePlan: IAirtableTablePlan;
+ tableIndex: number;
+ totalTables: number;
+ tableId: string;
+ usersByEmail: Map;
+ importAttachments: boolean;
+ recordIdMaps: Map>;
+ linkSpill: AirtableLinkRowSpill;
+ linkFieldsWithMulti: Set;
+ issues: IImportAirtableIssue[];
+ progress: (event: IAirtableImportProgress) => void;
+ }) {
+ const {
+ client,
+ ro,
+ tablePlan,
+ tableIndex,
+ totalTables,
+ tableId,
+ usersByEmail,
+ importAttachments,
+ recordIdMaps,
+ linkSpill,
+ linkFieldsWithMulti,
+ issues,
+ progress,
+ } = params;
+
+ progress({
+ phase: 'table_records_start',
+ tableName: tablePlan.name,
+ tableIndex,
+ totalTables,
+ });
+
+ const recordIdMap = new Map();
+ recordIdMaps.set(tablePlan.airtableTableId, recordIdMap);
+ const droppedCollaborators = new Map();
+ const failedAttachments = new Map();
+ let processedRows = 0;
+ let restarts = 0;
+
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ try {
+ for await (const page of client.listRecords(ro.airtableBaseId, tablePlan.airtableTableId)) {
+ // After an iterator restart, skip already imported records.
+ const records = page.filter((record) => !recordIdMap.has(record.id));
+ if (records.length === 0) continue;
+
+ const payloads = await this.buildRecordPayloads({
+ tablePlan,
+ records,
+ usersByEmail,
+ importAttachments,
+ droppedCollaborators,
+ failedAttachments,
+ });
+ // Teable assigns each create batch a descending order, which flips the
+ // rows in the view. Send the batch reversed so records keep the source
+ // order, then map results back by their original index.
+ const created = await this.recordOpenApiV2Service.createRecords(tableId, {
+ fieldKeyType: FieldKeyType.Id,
+ typecast: true,
+ records: payloads.slice().reverse(),
+ });
+ const createdInOrder = created.records.slice().reverse();
+ records.forEach((record, recordIndex) => {
+ const createdRecord = createdInOrder[recordIndex];
+ if (createdRecord) {
+ recordIdMap.set(record.id, createdRecord.id);
+ }
+ });
+ await this.collectLinkRows({
+ tablePlan,
+ records,
+ createdIds: createdInOrder.map((record) => record?.id),
+ linkSpill,
+ linkFieldsWithMulti,
+ });
+
+ processedRows += records.length;
+ progress({
+ phase: 'table_records_progress',
+ tableName: tablePlan.name,
+ tableIndex,
+ totalTables,
+ processedRows,
+ });
+ }
+ break;
+ } catch (error) {
+ if (error instanceof AirtableIteratorExpiredError && restarts < maxListRestarts) {
+ restarts++;
+ this.logger.warn(
+ `Airtable list-records iterator expired for table ${tablePlan.airtableTableId}, restarting (${restarts}/${maxListRestarts})`
+ );
+ continue;
+ }
+ throw error;
+ }
+ }
+
+ for (const [fieldName, count] of droppedCollaborators) {
+ issues.push({
+ code: 'valuesDropped',
+ tableName: tablePlan.name,
+ fieldName,
+ count,
+ reason: 'collaborator is not a member of the space',
+ });
+ }
+ for (const [fieldName, { count, firstError }] of failedAttachments) {
+ issues.push({
+ code: 'valuesDropped',
+ tableName: tablePlan.name,
+ fieldName,
+ count,
+ reason: `attachment download failed: ${firstError}`,
+ });
+ }
+
+ progress({
+ phase: 'table_records_done',
+ tableName: tablePlan.name,
+ tableIndex,
+ totalTables,
+ processedRows,
+ });
+ }
+
+ private async buildRecordPayloads(params: {
+ tablePlan: IAirtableTablePlan;
+ records: IAirtableRecord[];
+ usersByEmail: Map;
+ importAttachments: boolean;
+ droppedCollaborators: Map;
+ failedAttachments: Map;
+ }): Promise }>> {
+ const {
+ tablePlan,
+ records,
+ usersByEmail,
+ importAttachments,
+ droppedCollaborators,
+ failedAttachments,
+ } = params;
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+ const payloads = records.map((record) => {
+ const fields: Record = {};
+ for (const planned of tablePlan.fields) {
+ const raw = record.fields[planned.airtableFieldId];
+ if (raw == null) continue;
+ const fieldId = planned.ro.id as string;
+ if (planned.converter === 'user') {
+ const isMultiple = (planned.ro.options as IUserFieldOptions)?.isMultiple === true;
+ const { value, droppedCount } = convertCollaboratorCellValue(
+ raw,
+ usersByEmail,
+ isMultiple
+ );
+ if (droppedCount > 0) {
+ const name = planned.ro.name as string;
+ droppedCollaborators.set(name, (droppedCollaborators.get(name) ?? 0) + droppedCount);
+ }
+ if (value !== undefined) fields[fieldId] = value;
+ continue;
+ }
+ if (planned.converter === 'attachment') {
+ continue; // handled below, needs async upload
+ }
+ const value = convertAirtableCellValue(planned.converter, raw);
+ if (value !== undefined) fields[fieldId] = value;
+ }
+ return { fields };
+ });
+
+ if (importAttachments) {
+ const attachmentFields = tablePlan.fields.filter(
+ (planned) => planned.converter === 'attachment'
+ );
+ const cells: Array<{
+ payloadIndex: number;
+ fieldId: string;
+ fieldName: string;
+ attachments: IAirtableAttachment[];
+ }> = [];
+ records.forEach((record, payloadIndex) => {
+ for (const planned of attachmentFields) {
+ const raw = record.fields[planned.airtableFieldId];
+ if (!Array.isArray(raw) || raw.length === 0) continue;
+ cells.push({
+ payloadIndex,
+ fieldId: planned.ro.id as string,
+ fieldName: planned.ro.name as string,
+ attachments: raw as IAirtableAttachment[],
+ });
+ }
+ });
+
+ await mapWithConcurrency(cells, attachmentConcurrency, async (cell) => {
+ const items = [];
+ for (const attachment of cell.attachments) {
+ if (typeof attachment?.url !== 'string') continue;
+ try {
+ items.push(await this.transferAttachment(attachment));
+ } catch (error) {
+ const message = error instanceof Error ? error.message : unknownErrorText;
+ this.logger.warn(
+ `Failed to migrate Airtable attachment "${attachment.filename}": ${message}`
+ );
+ const failure = failedAttachments.get(cell.fieldName);
+ failedAttachments.set(cell.fieldName, {
+ count: (failure?.count ?? 0) + 1,
+ firstError: failure?.firstError ?? message.slice(0, 200),
+ });
+ }
+ }
+ if (items.length > 0) {
+ payloads[cell.payloadIndex].fields[cell.fieldId] = items;
+ }
+ });
+ }
+
+ return payloads;
+ }
+
+ /**
+ * Streams an attachment straight from the Airtable CDN into the storage
+ * backend - no temp file. Airtable provides the size up front, which the
+ * presigned upload requires.
+ */
+ private async transferAttachment(attachment: IAirtableAttachment) {
+ // Airtable gives the size up front, so reject an over-limit attachment
+ // before opening the CDN socket at all.
+ const maxSize = this.attachmentsService.thresholdConfig.maxOpenapiAttachmentUploadSize;
+ if (attachment.size && attachment.size > maxSize) {
+ throw new Error(
+ `attachment "${attachment.filename}" (${attachment.size} bytes) exceeds the ${maxSize}-byte upload limit`
+ );
+ }
+ const response = await fetch(attachment.url);
+ if (!response.ok || !response.body) {
+ throw new Error(`download failed with status ${response.status}`);
+ }
+ const contentLength = attachment.size ?? Number(response.headers.get('content-length'));
+ if (!contentLength || !Number.isFinite(contentLength)) {
+ await response.body.cancel();
+ throw new Error('attachment size is unknown');
+ }
+ const contentType =
+ attachment.type ?? response.headers.get('content-type') ?? 'application/octet-stream';
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const stream = Readable.fromWeb(response.body as any);
+ try {
+ return await this.attachmentsService.uploadFromStream(
+ stream,
+ { filename: attachment.filename, contentType, contentLength },
+ UploadType.Table
+ );
+ } catch (error) {
+ // Release the still-open CDN socket on any upload failure (e.g. the
+ // size-limit check) instead of leaking it.
+ stream.destroy();
+ throw error;
+ }
+ }
+
+ private async collectLinkRows(params: {
+ tablePlan: IAirtableTablePlan;
+ records: IAirtableRecord[];
+ createdIds: Array;
+ linkSpill: AirtableLinkRowSpill;
+ linkFieldsWithMulti: Set;
+ }) {
+ const { tablePlan, records, createdIds, linkSpill, linkFieldsWithMulti } = params;
+ if (tablePlan.linkFields.length === 0) return;
+
+ const rows: ISpilledLinkRow[] = [];
+ records.forEach((record, index) => {
+ const teableRecordId = createdIds[index];
+ if (!teableRecordId) return;
+ const cells: ISpilledLinkRow['cells'] = [];
+ for (const linkField of tablePlan.linkFields) {
+ const ids = extractLinkedRecordIds(record.fields[linkField.airtableFieldId]);
+ if (ids.length === 0) continue;
+ // Airtable's single-link is a soft per-cell limit, so a "single" field's
+ // data can still hold several links; remember those so the field is
+ // relaxed to many-to-many instead of truncating them.
+ if (ids.length > 1) linkFieldsWithMulti.add(linkField.airtableFieldId);
+ cells.push({ airtableFieldId: linkField.airtableFieldId, ids });
+ }
+ if (cells.length > 0) {
+ rows.push({ teableRecordId, cells });
+ }
+ });
+ await linkSpill.append(tablePlan.airtableTableId, rows);
+ }
+
+ /**
+ * Follows the relationship Airtable declares. One-to-* variants are not
+ * used because foreign-side uniqueness cannot be guaranteed before the
+ * records arrive; cells that violate a single link are truncated at fill
+ * time and reported.
+ */
+ private decideRelationship(linkField: IPlannedLinkField): Relationship {
+ return linkField.prefersSingle ? Relationship.ManyOne : Relationship.ManyMany;
+ }
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+ private async createLinkFields(params: {
+ plan: IAirtableImportPlan;
+ tableIdMap: Record;
+ viewIdMap: Record;
+ issues: IImportAirtableIssue[];
+ }): Promise> {
+ const { plan, tableIdMap, viewIdMap, issues } = params;
+ const runtimes = new Map();
+
+ for (const tablePlan of plan.tables) {
+ const tableId = tableIdMap[tablePlan.airtableTableId];
+ for (const linkField of tablePlan.linkFields) {
+ const relationship = this.decideRelationship(linkField);
+ // Airtable's "limit record selection to a view" -> Teable filterByViewId.
+ // The referenced view lives in the foreign table; skip (and report) when it
+ // did not import so an invalid view id is never set.
+ let filterByViewId: string | undefined;
+ if (linkField.viewIdForRecordSelection) {
+ filterByViewId = viewIdMap[linkField.viewIdForRecordSelection];
+ if (!filterByViewId) {
+ issues.push({
+ code: 'viewConfigDegraded',
+ tableName: tablePlan.name,
+ fieldName: linkField.name,
+ reason: 'record-selection view was not imported',
+ });
+ }
+ }
+ const fieldRo = {
+ id: linkField.teableFieldId,
+ name: linkField.name,
+ description: linkField.description,
+ type: FieldType.Link,
+ options: {
+ relationship,
+ foreignTableId: tableIdMap[linkField.airtableForeignTableId],
+ isOneWay: linkField.inverse == null,
+ ...(filterByViewId ? { filterByViewId } : {}),
+ },
+ } as IFieldRo;
+ try {
+ const created = await this.fieldOpenApiV2Service.createField(tableId, fieldRo);
+ plan.fieldIdMap[linkField.airtableFieldId] = created.id;
+ runtimes.set(linkField.airtableFieldId, {
+ plan: linkField,
+ tableAirtableId: tablePlan.airtableTableId,
+ relationship,
+ });
+
+ const symmetricFieldId = (created.options as ILinkFieldOptions)?.symmetricFieldId;
+ if (linkField.inverse && symmetricFieldId) {
+ plan.fieldIdMap[linkField.inverse.airtableFieldId] = symmetricFieldId;
+ try {
+ await this.renameSymmetricLinkField(
+ tableIdMap[linkField.airtableForeignTableId],
+ symmetricFieldId,
+ linkField.inverse.name
+ );
+ } catch (error) {
+ this.logger.warn(
+ `Failed to rename symmetric link field to "${linkField.inverse.name}": ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`
+ );
+ }
+ }
+ } catch (error) {
+ this.logger.warn(
+ `Failed to create link field "${linkField.name}": ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`
+ );
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName: linkField.name,
+ fromType: 'multipleRecordLinks',
+ reason: 'failed to create link field',
+ });
+ }
+ }
+ }
+ return runtimes;
+ }
+
+ /**
+ * Renames the symmetric link field Teable auto-creates for a two-way link to
+ * the Airtable inverse field's name. A plain field update drops the owning
+ * link field; convert keeps the link by re-specifying its full options.
+ */
+ private async renameSymmetricLinkField(tableId: string, fieldId: string, name: string) {
+ const symmetric = await this.fieldOpenApiV2Service.getField(tableId, fieldId);
+ await this.fieldOpenApiV2Service.convertField(tableId, fieldId, {
+ name,
+ type: symmetric.type,
+ options: symmetric.options,
+ } as Parameters[2]);
+ }
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+ private async createLookupFields(
+ plan: IAirtableImportPlan,
+ tableIdMap: Record,
+ issues: IImportAirtableIssue[]
+ ) {
+ for (const tablePlan of plan.tables) {
+ const tableId = tableIdMap[tablePlan.airtableTableId];
+ for (const lookupField of tablePlan.lookupFields) {
+ const foreignPlan = plan.tables.find(
+ (candidate) => candidate.airtableTableId === lookupField.airtableForeignTableId
+ );
+ const targetField = foreignPlan?.fields.find(
+ (candidate) => candidate.airtableFieldId === lookupField.airtableTargetFieldId
+ );
+ const linkFieldId = plan.fieldIdMap[lookupField.airtableLinkFieldId];
+ const lookupFieldId = plan.fieldIdMap[lookupField.airtableTargetFieldId];
+ if (!targetField || !linkFieldId || !lookupFieldId) {
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName: lookupField.name,
+ fromType: 'multipleLookupValues',
+ reason: 'link or target field was not imported',
+ });
+ continue;
+ }
+ const fieldRo = {
+ id: lookupField.teableFieldId,
+ name: lookupField.name,
+ description: lookupField.description,
+ type: targetField.ro.type,
+ isLookup: true,
+ lookupOptions: {
+ foreignTableId: tableIdMap[lookupField.airtableForeignTableId],
+ linkFieldId,
+ lookupFieldId,
+ },
+ } as IFieldRo;
+ try {
+ await this.fieldOpenApiV2Service.createField(tableId, fieldRo);
+ } catch (error) {
+ this.logger.warn(
+ `Failed to create lookup field "${lookupField.name}": ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`
+ );
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName: lookupField.name,
+ fromType: 'multipleLookupValues',
+ reason: `failed to create lookup field: ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`,
+ });
+ }
+ }
+ }
+ }
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+ private async createCountFields(
+ plan: IAirtableImportPlan,
+ tableIdMap: Record,
+ issues: IImportAirtableIssue[]
+ ) {
+ for (const tablePlan of plan.tables) {
+ const tableId = tableIdMap[tablePlan.airtableTableId];
+ for (const countField of tablePlan.countFields) {
+ const foreignPlan = plan.tables.find(
+ (candidate) => candidate.airtableTableId === countField.airtableForeignTableId
+ );
+ const linkFieldId = plan.fieldIdMap[countField.airtableLinkFieldId];
+ const foreignPrimary = foreignPlan?.fields[0];
+ if (!linkFieldId || !foreignPrimary) {
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName: countField.name,
+ fromType: 'count',
+ reason: 'link field was not imported',
+ });
+ continue;
+ }
+ const fieldRo = {
+ id: countField.teableFieldId,
+ name: countField.name,
+ description: countField.description,
+ type: FieldType.Rollup,
+ options: { expression: 'countall({values})' },
+ lookupOptions: {
+ foreignTableId: tableIdMap[countField.airtableForeignTableId],
+ linkFieldId,
+ lookupFieldId: foreignPrimary.ro.id as string,
+ },
+ } as IFieldRo;
+ try {
+ await this.fieldOpenApiV2Service.createField(tableId, fieldRo);
+ } catch (error) {
+ this.logger.warn(
+ `Failed to create count field "${countField.name}": ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`
+ );
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName: countField.name,
+ fromType: 'count',
+ reason: `failed to create rollup field: ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`,
+ });
+ }
+ }
+ }
+ }
+
+ /**
+ * Recreates rollups as live Teable rollups using the aggregation and optional
+ * record-selection filter read from the shared base model. The filter is mapped
+ * best-effort — conditions that cannot be translated are dropped (and reported)
+ * so the rollup stays live rather than degrading to a snapshot.
+ */
+ // eslint-disable-next-line sonarjs/cognitive-complexity -- per-rollup filter mapping
+ private async createRollupFields(
+ plan: IAirtableImportPlan,
+ tableIdMap: Record,
+ airtableTables: IAirtableTable[],
+ issues: IImportAirtableIssue[]
+ ) {
+ const rollups = plan.tables.flatMap((tablePlan) =>
+ tablePlan.rollupFields.map((rollupField) => ({ tablePlan, rollupField }))
+ );
+ if (rollups.length === 0) return;
+ const optionNames = this.buildSelectOptionNames(airtableTables);
+
+ for (const { tablePlan, rollupField } of rollups) {
+ const linkFieldId = plan.fieldIdMap[rollupField.airtableLinkFieldId];
+ const lookupFieldId = plan.fieldIdMap[rollupField.airtableForeignFieldId];
+ const foreignTableId = tableIdMap[rollupField.airtableForeignTableId];
+ if (!linkFieldId || !lookupFieldId || !foreignTableId) {
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName: rollupField.name,
+ fromType: 'rollup',
+ reason: 'link or rolled-up field was not imported',
+ });
+ continue;
+ }
+
+ let filter: IFilter | undefined;
+ if (rollupField.filter) {
+ const metaMap = await this.fetchFieldMeta(foreignTableId);
+ let dropped = 0;
+ const ctx: IViewConfigMapperContext = {
+ resolveField: (columnId) => {
+ const teableFieldId = plan.fieldIdMap[columnId];
+ return teableFieldId ? metaMap.get(teableFieldId) : undefined;
+ },
+ resolveSelectOptionName: (columnId, optionId) => optionNames.get(columnId)?.get(optionId),
+ };
+ filter = mapAirtableFilter(rollupField.filter, ctx, () => {
+ dropped += 1;
+ });
+ if (dropped > 0) {
+ issues.push({
+ code: 'fieldDegraded',
+ tableName: tablePlan.name,
+ fieldName: rollupField.name,
+ fromType: 'rollup',
+ toType: `live rollup (${dropped} filter condition(s) dropped)`,
+ });
+ }
+ }
+
+ const fieldRo = {
+ id: rollupField.teableFieldId,
+ name: rollupField.name,
+ description: rollupField.description,
+ type: FieldType.Rollup,
+ options: { expression: rollupField.expression },
+ lookupOptions: {
+ foreignTableId,
+ linkFieldId,
+ lookupFieldId,
+ ...(filter ? { filter } : {}),
+ },
+ } as IFieldRo;
+ try {
+ await this.fieldOpenApiV2Service.createField(
+ tableIdMap[tablePlan.airtableTableId],
+ fieldRo
+ );
+ } catch (error) {
+ this.logger.warn(
+ `Failed to create rollup field "${rollupField.name}": ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`
+ );
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName: rollupField.name,
+ fromType: 'rollup',
+ reason: `failed to create rollup field: ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`,
+ });
+ }
+ }
+ }
+
+ /**
+ * Creates the translated formula fields last, after every other field exists.
+ * A formula may reference another formula, so fields are created in dependency
+ * passes: each pass retries the ones that failed (typically because a
+ * referenced formula did not exist yet) until a pass makes no progress.
+ */
+ // eslint-disable-next-line sonarjs/cognitive-complexity -- dependency-pass retry loop
+ private async createFormulaFields(
+ plan: IAirtableImportPlan,
+ tableIdMap: Record,
+ issues: IImportAirtableIssue[]
+ ) {
+ const remapReferences = (expression: string) =>
+ expression.replace(/\{(fld[a-zA-Z0-9]+)\}/g, (token, airtableFieldId) => {
+ const teableFieldId = plan.fieldIdMap[airtableFieldId];
+ return teableFieldId ? `{${teableFieldId}}` : token;
+ });
+
+ let pending = plan.tables.flatMap((tablePlan) =>
+ tablePlan.formulaFields.map((formulaField) => ({ tablePlan, formulaField }))
+ );
+ while (pending.length > 0) {
+ const failed: {
+ tablePlan: IAirtableTablePlan;
+ formulaField: IPlannedFormulaField;
+ error: unknown;
+ }[] = [];
+ for (const { tablePlan, formulaField } of pending) {
+ const fieldRo = {
+ id: formulaField.teableFieldId,
+ name: formulaField.name,
+ description: formulaField.description,
+ type: FieldType.Formula,
+ options: { expression: remapReferences(formulaField.expression) },
+ } as IFieldRo;
+ try {
+ await this.fieldOpenApiV2Service.createField(
+ tableIdMap[tablePlan.airtableTableId],
+ fieldRo
+ );
+ } catch (error) {
+ failed.push({ tablePlan, formulaField, error });
+ }
+ }
+ if (failed.length === pending.length) {
+ // No progress this pass: the remaining failures are real, report them.
+ for (const { tablePlan, formulaField, error } of failed) {
+ this.logger.warn(
+ `Failed to create formula field "${formulaField.name}": ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`
+ );
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName: formulaField.name,
+ fromType: 'formula',
+ reason: `failed to create formula field: ${
+ error instanceof Error ? error.message : unknownErrorText
+ }`,
+ });
+ }
+ break;
+ }
+ pending = failed.map(({ tablePlan, formulaField }) => ({ tablePlan, formulaField }));
+ }
+ }
+
+ /**
+ * Airtable's "single record link" is a soft per-cell limit, not enforced
+ * uniqueness — a single-link field's data can still hold several links (e.g.
+ * the side of a 1:1 that several records point at). Teable's ManyOne would
+ * truncate those, so relax any single link whose data actually held multiple
+ * values to a many-to-many link (lossless) and report it, instead of dropping
+ * links. Runs before fill, while the field is still empty, so the conversion
+ * is trivially safe; a failed conversion falls back to truncate-and-report.
+ */
+ private async relaxOversizedSingleLinks(params: {
+ plan: IAirtableImportPlan;
+ linkRuntimes: Map;
+ linkFieldsWithMulti: Set;
+ tableIdMap: Record;
+ issues: IImportAirtableIssue[];
+ }): Promise {
+ const { plan, linkRuntimes, linkFieldsWithMulti, tableIdMap, issues } = params;
+ const tableNameByAirtableId = new Map(plan.tables.map((t) => [t.airtableTableId, t.name]));
+ for (const runtime of linkRuntimes.values()) {
+ if (runtime.relationship !== Relationship.ManyOne) continue;
+ if (!linkFieldsWithMulti.has(runtime.plan.airtableFieldId)) continue;
+ const tableId = tableIdMap[runtime.tableAirtableId];
+ try {
+ await this.fieldOpenApiV2Service.convertField(tableId, runtime.plan.teableFieldId, {
+ name: runtime.plan.name,
+ type: FieldType.Link,
+ options: {
+ relationship: Relationship.ManyMany,
+ foreignTableId: tableIdMap[runtime.plan.airtableForeignTableId],
+ isOneWay: runtime.plan.inverse == null,
+ },
+ } as Parameters[2]);
+ runtime.relationship = Relationship.ManyMany;
+ issues.push({
+ code: 'fieldDegraded',
+ tableName: tableNameByAirtableId.get(runtime.tableAirtableId) ?? '',
+ fieldName: runtime.plan.name,
+ fromType: 'multipleRecordLinks',
+ toType: 'many-to-many link',
+ reason: 'single-link data contained multiple links; kept them all',
+ });
+ } catch (error) {
+ // Keep ManyOne — fillLinkValues truncates the over-capacity cells and
+ // reports them, so the import still succeeds.
+ this.logger.warn(
+ `[airtable-import] could not relax single link "${runtime.plan.name}": ${
+ error instanceof Error ? error.message : 'error'
+ }`
+ );
+ }
+ }
+ }
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+ private async fillLinkValues(params: {
+ plan: IAirtableImportPlan;
+ tableIdMap: Record;
+ linkRuntimes: Map;
+ recordIdMaps: Map>;
+ linkSpill: AirtableLinkRowSpill;
+ issues: IImportAirtableIssue[];
+ progress: (event: IAirtableImportProgress) => void;
+ }) {
+ const { plan, tableIdMap, linkRuntimes, recordIdMaps, linkSpill, issues, progress } = params;
+
+ for (const tablePlan of plan.tables) {
+ if (tablePlan.linkFields.length === 0) continue;
+ const tableId = tableIdMap[tablePlan.airtableTableId];
+ // Only write link fields that actually exist; one that failed to
+ // materialize is skipped and reported, never fatal to the import.
+ const existingFieldIds = new Set((await this.fetchFieldMeta(tableId)).keys());
+ const droppedByField = new Map();
+ const truncatedByField = new Map();
+ const missingByField = new Set();
+ let processedRows = 0;
+ let batch: Array<{ id: string; fields: Record }> = [];
+
+ const flush = async () => {
+ if (batch.length === 0) return;
+ await this.recordOpenApiV2Service.updateRecords(tableId, {
+ fieldKeyType: FieldKeyType.Id,
+ typecast: false,
+ records: batch,
+ });
+ batch = [];
+ progress({ phase: 'filling_links', tableName: tablePlan.name, processedRows });
+ };
+
+ // Rows stream back from the disk spill, so memory stays flat no matter
+ // how many link cells the base contains.
+ for await (const row of linkSpill.read(tablePlan.airtableTableId)) {
+ processedRows++;
+ const fields: Record = {};
+ for (const cell of row.cells) {
+ const runtime = linkRuntimes.get(cell.airtableFieldId);
+ const teableFieldId = plan.fieldIdMap[cell.airtableFieldId];
+ if (!runtime || !teableFieldId) continue;
+ if (!existingFieldIds.has(teableFieldId)) {
+ missingByField.add(runtime.plan.name);
+ continue;
+ }
+ const foreignMap = recordIdMaps.get(runtime.plan.airtableForeignTableId);
+ const mappedIds = cell.ids
+ .map((id) => foreignMap?.get(id))
+ .filter((id): id is string => id != null);
+ const droppedCount = cell.ids.length - mappedIds.length;
+ if (droppedCount > 0) {
+ droppedByField.set(
+ runtime.plan.name,
+ (droppedByField.get(runtime.plan.name) ?? 0) + droppedCount
+ );
+ }
+ if (mappedIds.length === 0) continue;
+ const isSingle = runtime.relationship === Relationship.ManyOne;
+ if (isSingle && mappedIds.length > 1) {
+ // The Airtable field declared single links but the data disagrees.
+ truncatedByField.set(
+ runtime.plan.name,
+ (truncatedByField.get(runtime.plan.name) ?? 0) + mappedIds.length - 1
+ );
+ }
+ fields[teableFieldId] = isSingle ? { id: mappedIds[0] } : mappedIds.map((id) => ({ id }));
+ }
+ if (Object.keys(fields).length > 0) {
+ batch.push({ id: row.teableRecordId, fields });
+ if (batch.length >= linkUpdateBatchSize) {
+ await flush();
+ }
+ }
+ }
+ await flush();
+
+ for (const [fieldName, count] of droppedByField) {
+ issues.push({
+ code: 'valuesDropped',
+ tableName: tablePlan.name,
+ fieldName,
+ count,
+ reason: 'linked record not found',
+ });
+ }
+ for (const [fieldName, count] of truncatedByField) {
+ issues.push({
+ code: 'valuesDropped',
+ tableName: tablePlan.name,
+ fieldName,
+ count,
+ reason: 'kept only the first record of a single-link field',
+ });
+ }
+ for (const fieldName of missingByField) {
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: tablePlan.name,
+ fieldName,
+ fromType: 'multipleRecordLinks',
+ reason: 'link field was not created; its values were skipped',
+ });
+ }
+ }
+ }
+}
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-link-spill.spec.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-link-spill.spec.ts
new file mode 100644
index 0000000000..a87510792c
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-link-spill.spec.ts
@@ -0,0 +1,65 @@
+import { Readable } from 'node:stream';
+import { describe, expect, it } from 'vitest';
+import {
+ AirtableLinkRowSpill,
+ type ISpillStorage,
+ type ISpilledLinkRow,
+} from './airtable-link-spill';
+
+const row = (recordId: string, ids: string[]): ISpilledLinkRow => ({
+ teableRecordId: recordId,
+ cells: [{ airtableFieldId: 'fldLink', ids }],
+});
+
+const createMemoryStorage = () => {
+ const objects = new Map();
+ const storage: ISpillStorage = {
+ upload: async (path, data) => {
+ objects.set(path, data);
+ },
+ download: async (path) => Readable.from(objects.get(path) ?? Buffer.alloc(0)),
+ cleanup: async (dir) => {
+ for (const key of [...objects.keys()]) {
+ if (key.startsWith(dir)) objects.delete(key);
+ }
+ },
+ };
+ return { storage, objects };
+};
+
+describe('AirtableLinkRowSpill', () => {
+ it('fails with a clear error past the staging budget', async () => {
+ const { storage } = createMemoryStorage();
+ const spill = new AirtableLinkRowSpill(storage, 16);
+ await expect(
+ spill.append('tblA', [row('rec1', ['recX']), row('rec2', ['recY'])])
+ ).rejects.toThrow(/TEABLE_IMPORT_SPILL_MAX_BYTES/);
+ });
+
+ it('streams appended rows back per table in order and cleans up', async () => {
+ const { storage, objects } = createMemoryStorage();
+ const spill = new AirtableLinkRowSpill(storage);
+ await spill.append('tblA', [row('rec1', ['recX']), row('rec2', ['recY', 'recZ'])]);
+ await spill.append('tblA', [row('rec3', ['recX'])]);
+ await spill.append('tblB', [row('rec9', ['recQ'])]);
+
+ const readAll = async (tableId: string) => {
+ const rows: ISpilledLinkRow[] = [];
+ for await (const item of spill.read(tableId)) {
+ rows.push(item);
+ }
+ return rows;
+ };
+
+ const tableA = await readAll('tblA');
+ expect(tableA.map((item) => item.teableRecordId)).toEqual(['rec1', 'rec2', 'rec3']);
+ expect(tableA[1].cells[0].ids).toEqual(['recY', 'recZ']);
+ expect(await readAll('tblB')).toHaveLength(1);
+ // A table that never buffered anything yields nothing.
+ expect(await readAll('tblC')).toHaveLength(0);
+
+ expect(objects.size).toBeGreaterThan(0);
+ await spill.cleanup();
+ expect(objects.size).toBe(0);
+ });
+});
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-link-spill.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-link-spill.ts
new file mode 100644
index 0000000000..2fe842fddf
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-link-spill.ts
@@ -0,0 +1,110 @@
+import { randomUUID } from 'node:crypto';
+import { createInterface } from 'node:readline';
+import type { Readable } from 'node:stream';
+
+export interface ISpilledLinkRow {
+ teableRecordId: string;
+ cells: Array<{ airtableFieldId: string; ids: string[] }>;
+}
+
+/**
+ * Minimal IO the spill needs; the importer backs it with the deployment's
+ * StorageAdapter (local, S3 or MinIO) — the same staging pattern the .tea
+ * import uses, no container-local temp files.
+ */
+export interface ISpillStorage {
+ upload(path: string, data: Buffer): Promise;
+ download(path: string): Promise;
+ /** Removes everything under the given directory prefix. */
+ cleanup(dir: string): Promise;
+}
+
+const defaultSpillMaxBytes = 2 * 1024 * 1024 * 1024; // 2 GiB
+const partMaxBytes = 4 * 1024 * 1024; // 4 MiB per uploaded part
+
+const getSpillMaxBytes = () => {
+ const fromEnv = Number(process.env.TEABLE_IMPORT_SPILL_MAX_BYTES);
+ return Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : defaultSpillMaxBytes;
+};
+
+/**
+ * Staging area for the link cells collected during record import. Link values
+ * can only be written after every table's records exist (the old->new record
+ * id mapping must be complete), so rows are buffered — as JSONL parts in the
+ * blob storage, keeping the importer's memory flat for large bases.
+ */
+export class AirtableLinkRowSpill {
+ private readonly dir = `airtable-import/${randomUUID()}`;
+ private readonly pending = new Map();
+ private readonly parts = new Map();
+ private readonly maxBytes: number;
+ private bytesWritten = 0;
+ private uploadedAnything = false;
+
+ constructor(
+ private readonly storage: ISpillStorage,
+ maxBytes = getSpillMaxBytes()
+ ) {
+ this.maxBytes = maxBytes;
+ }
+
+ async append(airtableTableId: string, rows: ISpilledLinkRow[]): Promise {
+ if (rows.length === 0) return;
+ let pending = this.pending.get(airtableTableId);
+ if (!pending) {
+ pending = { lines: [], bytes: 0 };
+ this.pending.set(airtableTableId, pending);
+ }
+ let addedBytes = 0;
+ for (const row of rows) {
+ const line = `${JSON.stringify(row)}\n`;
+ pending.lines.push(line);
+ addedBytes += Buffer.byteLength(line);
+ }
+ pending.bytes += addedBytes;
+ this.bytesWritten += addedBytes;
+ if (this.bytesWritten > this.maxBytes) {
+ throw new Error(
+ `The import link buffer exceeded ${this.maxBytes} bytes of staging storage; ` +
+ 'raise TEABLE_IMPORT_SPILL_MAX_BYTES to import this base'
+ );
+ }
+ if (pending.bytes >= partMaxBytes) {
+ await this.flushPart(airtableTableId);
+ }
+ }
+
+ private async flushPart(airtableTableId: string): Promise {
+ const pending = this.pending.get(airtableTableId);
+ if (!pending || pending.lines.length === 0) return;
+ const parts = this.parts.get(airtableTableId) ?? [];
+ const partPath = `${this.dir}/${airtableTableId}.part-${String(parts.length).padStart(5, '0')}.jsonl`;
+ await this.storage.upload(partPath, Buffer.from(pending.lines.join('')));
+ parts.push(partPath);
+ this.parts.set(airtableTableId, parts);
+ this.pending.delete(airtableTableId);
+ this.uploadedAnything = true;
+ }
+
+ /** Streams the table's rows back, part by part, line by line. */
+ async *read(airtableTableId: string): AsyncGenerator {
+ await this.flushPart(airtableTableId);
+ for (const partPath of this.parts.get(airtableTableId) ?? []) {
+ const stream = await this.storage.download(partPath);
+ const lines = createInterface({ input: stream, crlfDelay: Infinity });
+ for await (const line of lines) {
+ if (line.trim()) {
+ yield JSON.parse(line) as ISpilledLinkRow;
+ }
+ }
+ }
+ }
+
+ async cleanup(): Promise {
+ this.pending.clear();
+ this.parts.clear();
+ if (this.uploadedAnything) {
+ await this.storage.cleanup(this.dir);
+ }
+ }
+}
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-record-converter.spec.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-record-converter.spec.ts
new file mode 100644
index 0000000000..a8d7dfe396
Binary files /dev/null and b/apps/nestjs-backend/src/features/airtable-import/airtable-record-converter.spec.ts differ
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-record-converter.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-record-converter.ts
new file mode 100644
index 0000000000..9d5cc06f32
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-record-converter.ts
@@ -0,0 +1,171 @@
+import type { IAirtableCellConverter } from './airtable-schema-mapper';
+import type { IAirtableCollaborator } from './airtable.types';
+
+/**
+ * Computed Airtable cells can carry error markers instead of values, e.g.
+ * `{"error": "#ERROR!"}` or `{"specialValue": "NaN"}`.
+ */
+const isErrorValue = (value: unknown): boolean =>
+ typeof value === 'object' &&
+ value !== null &&
+ !Array.isArray(value) &&
+ ('error' in value || 'specialValue' in value);
+
+const sanitizeString = (value: string) => value.replace(/\0/g, '');
+
+const toDisplayString = (value: unknown): string | undefined => {
+ if (value == null || isErrorValue(value)) return undefined;
+ if (typeof value === 'string') return sanitizeString(value);
+ if (typeof value === 'number' || typeof value === 'boolean') return String(value);
+ if (Array.isArray(value)) {
+ const parts = value.map(toDisplayString).filter((part): part is string => part != null);
+ return parts.length > 0 ? parts.join(', ') : undefined;
+ }
+ if (typeof value === 'object') {
+ const record = value as Record;
+ // Attachment-like, collaborator-like and button-like objects.
+ const label = record.filename ?? record.name ?? record.label ?? record.text ?? record.email;
+ if (typeof label === 'string') return sanitizeString(label);
+ return undefined;
+ }
+ return undefined;
+};
+
+const toNumber = (value: unknown): number | undefined => {
+ if (typeof value === 'number' && Number.isFinite(value)) return value;
+ if (typeof value === 'string') {
+ const parsed = Number(value);
+ return Number.isFinite(parsed) ? parsed : undefined;
+ }
+ return undefined;
+};
+
+const collaboratorToText = (value: IAirtableCollaborator): string | undefined => {
+ if (value.name && value.email) return `${value.name} (${value.email})`;
+ return value.name ?? value.email ?? undefined;
+};
+
+/**
+ * Converts a raw Airtable cell value (`cellFormat=json`) to a Teable cell
+ * value for the given converter kind. Returns `undefined` when the cell
+ * should stay empty. `user` and `attachment` cells need external context
+ * (collaborator resolution, file re-upload) and are handled by the importer.
+ */
+// eslint-disable-next-line sonarjs/cognitive-complexity
+export function convertAirtableCellValue(converter: IAirtableCellConverter, raw: unknown): unknown {
+ if (raw == null) return undefined;
+ switch (converter) {
+ case 'string':
+ return typeof raw === 'string' ? sanitizeString(raw) || undefined : toDisplayString(raw);
+ case 'number':
+ return isErrorValue(raw) ? undefined : toNumber(raw);
+ case 'boolean':
+ return raw === true ? true : undefined;
+ case 'dateTime':
+ return typeof raw === 'string' && !Number.isNaN(Date.parse(raw)) ? raw : undefined;
+ case 'stringArray': {
+ const values = Array.isArray(raw) ? raw : [raw];
+ const names = values.filter((item): item is string => typeof item === 'string');
+ return names.length > 0 ? names : undefined;
+ }
+ case 'barcode':
+ return typeof (raw as { text?: unknown }).text === 'string'
+ ? sanitizeString((raw as { text: string }).text)
+ : undefined;
+ case 'button': {
+ const { label, url } = raw as { label?: unknown; url?: unknown };
+ const labelText = typeof label === 'string' ? label : undefined;
+ const urlText = typeof url === 'string' ? url : undefined;
+ if (labelText && urlText) return `${labelText} (${urlText})`;
+ return labelText ?? urlText ?? undefined;
+ }
+ case 'aiText': {
+ const value = (raw as { value?: unknown }).value;
+ return typeof value === 'string' ? sanitizeString(value) || undefined : undefined;
+ }
+ case 'collaboratorText': {
+ const collaborators = (Array.isArray(raw) ? raw : [raw]) as IAirtableCollaborator[];
+ const parts = collaborators
+ .filter((item) => typeof item === 'object' && item !== null)
+ .map(collaboratorToText)
+ .filter((part): part is string => part != null);
+ return parts.length > 0 ? parts.join('; ') : undefined;
+ }
+ case 'snapshotText':
+ return toDisplayString(raw);
+ case 'snapshotNumber': {
+ if (isErrorValue(raw)) return undefined;
+ if (Array.isArray(raw)) return toNumber(raw[0]);
+ return toNumber(raw);
+ }
+ case 'snapshotDate': {
+ const value = Array.isArray(raw) ? raw[0] : raw;
+ return typeof value === 'string' && !Number.isNaN(Date.parse(value)) ? value : undefined;
+ }
+ case 'snapshotCheckbox': {
+ const value = Array.isArray(raw) ? raw[0] : raw;
+ return value === true ? true : undefined;
+ }
+ // Resolved asynchronously by the importer.
+ case 'user':
+ case 'attachment':
+ return undefined;
+ default:
+ return undefined;
+ }
+}
+
+export interface IResolvedSpaceUser {
+ id: string;
+ name: string;
+ email: string;
+}
+
+/**
+ * Converts Airtable collaborator cells to Teable user cell values by matching
+ * collaborator emails against the target space's collaborators. Returns the
+ * number of collaborators that could not be matched (and were dropped).
+ */
+export const convertCollaboratorCellValue = (
+ raw: unknown,
+ usersByEmail: Map,
+ isMultiple: boolean
+): { value: unknown; droppedCount: number } => {
+ const collaborators = (Array.isArray(raw) ? raw : [raw]).filter(
+ (item): item is IAirtableCollaborator => typeof item === 'object' && item !== null
+ );
+ const resolved = [];
+ let droppedCount = 0;
+ for (const collaborator of collaborators) {
+ const user = collaborator.email
+ ? usersByEmail.get(collaborator.email.toLowerCase())
+ : undefined;
+ if (user) {
+ resolved.push({ id: user.id, title: user.name, email: user.email });
+ } else {
+ droppedCount++;
+ }
+ }
+ if (resolved.length === 0) {
+ return { value: undefined, droppedCount };
+ }
+ return { value: isMultiple ? resolved : resolved[0], droppedCount };
+};
+
+/** Extracts linked record ids from a multipleRecordLinks cell. */
+export const extractLinkedRecordIds = (raw: unknown): string[] => {
+ if (!Array.isArray(raw)) return [];
+ return raw
+ .map((item) => {
+ if (typeof item === 'string') return item;
+ if (
+ typeof item === 'object' &&
+ item !== null &&
+ typeof (item as { id?: unknown }).id === 'string'
+ ) {
+ return (item as { id: string }).id;
+ }
+ return undefined;
+ })
+ .filter((id): id is string => id != null);
+};
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-rollup-mapper.spec.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-rollup-mapper.spec.ts
new file mode 100644
index 0000000000..31be09a914
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-rollup-mapper.spec.ts
@@ -0,0 +1,43 @@
+import { mapAirtableRollupAggregation } from './airtable-rollup-mapper';
+
+describe('mapAirtableRollupAggregation', () => {
+ it('maps every Airtable rollup function to its Teable counterpart', () => {
+ const cases: Array<[string, string]> = [
+ ['SUM(values)', 'sum({values})'],
+ ['AVERAGE(values)', 'average({values})'],
+ ['MAX(values)', 'max({values})'],
+ ['MIN(values)', 'min({values})'],
+ ['COUNT(values)', 'count({values})'],
+ ['COUNTA(values)', 'counta({values})'],
+ ['COUNTALL(values)', 'countall({values})'],
+ ['AND(values)', 'and({values})'],
+ ['OR(values)', 'or({values})'],
+ ['XOR(values)', 'xor({values})'],
+ ['ARRAYUNIQUE(values)', 'array_unique({values})'],
+ ['ARRAYCOMPACT(values)', 'array_compact({values})'],
+ ['CONCATENATE(values)', 'concatenate({values})'],
+ ];
+ for (const [airtable, teable] of cases) {
+ expect(mapAirtableRollupAggregation(airtable)).toBe(teable);
+ }
+ });
+
+ it("drops ARRAYJOIN's separator argument (Teable joins with a comma)", () => {
+ expect(mapAirtableRollupAggregation('ARRAYJOIN(values)')).toBe('array_join({values})');
+ expect(mapAirtableRollupAggregation('ARRAYJOIN(values, "; ")')).toBe('array_join({values})');
+ });
+
+ it('is case-insensitive and tolerates whitespace', () => {
+ expect(mapAirtableRollupAggregation(' sum( values ) ')).toBe('sum({values})');
+ });
+
+ it('returns null for compound or custom aggregations so the importer snapshots them', () => {
+ expect(mapAirtableRollupAggregation('SUM(values) * 2')).toBeNull();
+ expect(mapAirtableRollupAggregation('ROUND(AVERAGE(values), 1)')).toBeNull();
+ expect(mapAirtableRollupAggregation('SUM(values) / COUNTA(values)')).toBeNull();
+ });
+
+ it('returns null for an unsupported function', () => {
+ expect(mapAirtableRollupAggregation('MEDIAN(values)')).toBeNull();
+ });
+});
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-rollup-mapper.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-rollup-mapper.ts
new file mode 100644
index 0000000000..1d3dcfabce
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-rollup-mapper.ts
@@ -0,0 +1,43 @@
+/**
+ * Maps an Airtable rollup aggregation (the `formulaTextParsed` of a rollup
+ * column, e.g. "SUM(values)") to a Teable rollup expression ("sum({values})").
+ *
+ * Teable rollups accept a fixed set of aggregation functions, every one of which
+ * has a direct Airtable counterpart, so a single `FUNC(values)` aggregation
+ * translates 1:1. Anything outside that — a compound or custom aggregation such
+ * as "SUM(values) * 2" or "ROUND(AVERAGE(values), 1)" — has no live equivalent
+ * and returns null, so the importer falls back to a static value snapshot.
+ */
+
+/** Airtable rollup function (upper-case) → Teable rollup function. */
+const airtableRollupToTeable: Record = {
+ SUM: 'sum',
+ AVERAGE: 'average',
+ MAX: 'max',
+ MIN: 'min',
+ COUNT: 'count',
+ COUNTA: 'counta',
+ COUNTALL: 'countall',
+ AND: 'and',
+ OR: 'or',
+ XOR: 'xor',
+ ARRAYJOIN: 'array_join',
+ ARRAYUNIQUE: 'array_unique',
+ ARRAYCOMPACT: 'array_compact',
+ CONCATENATE: 'concatenate',
+};
+
+// One `FUNC(values)` call and nothing else. ARRAYJOIN also accepts a trailing
+// separator argument, which is dropped: Teable's array_join joins with a comma.
+const singleAggregation = /^([a-z]+)\s*\(\s*values\s*(?:,[^)]*)?\)$/i;
+
+/**
+ * Returns the Teable rollup expression for an Airtable aggregation, or null when
+ * the aggregation is compound/custom and cannot be a live Teable rollup.
+ */
+export const mapAirtableRollupAggregation = (formulaTextParsed: string): string | null => {
+ const match = formulaTextParsed.trim().match(singleAggregation);
+ if (!match) return null;
+ const teableFn = airtableRollupToTeable[match[1].toUpperCase()];
+ return teableFn ? `${teableFn}({values})` : null;
+};
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-schema-mapper.spec.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-schema-mapper.spec.ts
new file mode 100644
index 0000000000..ada44180c6
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-schema-mapper.spec.ts
@@ -0,0 +1,454 @@
+import { createFieldRoSchema, FieldType, ViewType } from '@teable/core';
+import { describe, expect, it } from 'vitest';
+import { buildAirtableImportPlan } from './airtable-schema-mapper';
+import type { IAirtableTable } from './airtable.types';
+
+const projectsLinkFieldId = 'fldLinkTasks000001';
+const tasksLinkFieldId = 'fldLinkProject0001';
+
+const buildSchema = (): IAirtableTable[] => [
+ {
+ id: 'tblProjects',
+ name: 'Projects',
+ primaryFieldId: 'fldName',
+ fields: [
+ { id: 'fldName', name: 'Name', type: 'singleLineText' },
+ { id: 'fldNotes', name: 'Notes', type: 'richText' },
+ {
+ id: 'fldBudget',
+ name: 'Budget',
+ type: 'currency',
+ options: { precision: 2, symbol: '€' },
+ },
+ {
+ id: 'fldDone',
+ name: 'Done',
+ type: 'checkbox',
+ options: { icon: 'check', color: 'greenBright' },
+ },
+ {
+ id: 'fldPriority',
+ name: 'Priority',
+ type: 'singleSelect',
+ options: {
+ choices: [
+ { id: 'selA', name: 'High', color: 'redBright' },
+ { id: 'selB', name: 'Low' },
+ { id: 'selC', name: '' },
+ ],
+ },
+ },
+ {
+ id: 'fldDue',
+ name: 'Due',
+ type: 'dateTime',
+ options: {
+ timeZone: 'client',
+ dateFormat: { name: 'european' },
+ timeFormat: { name: '24hour' },
+ },
+ },
+ {
+ id: 'fldScore',
+ name: 'Score',
+ type: 'formula',
+ options: {
+ formula: '{fldBudget} * 2',
+ isValid: true,
+ result: { type: 'number', options: { precision: 1 } },
+ },
+ },
+ { id: 'fldOwner', name: 'Owner', type: 'singleCollaborator' },
+ { id: 'fldFiles', name: 'Files', type: 'multipleAttachments' },
+ {
+ id: projectsLinkFieldId,
+ name: 'Tasks',
+ type: 'multipleRecordLinks',
+ options: {
+ linkedTableId: 'tblTasks',
+ prefersSingleRecordLink: false,
+ inverseLinkFieldId: tasksLinkFieldId,
+ isReversed: false,
+ },
+ },
+ {
+ id: 'fldTaskCount',
+ name: 'Task count',
+ type: 'count',
+ options: { recordLinkFieldId: projectsLinkFieldId, isValid: true },
+ },
+ {
+ id: 'fldTaskRollup',
+ name: 'Task rollup',
+ type: 'rollup',
+ // The aggregation is not in the official API options; it arrives via rollupSources.
+ options: {
+ recordLinkFieldId: projectsLinkFieldId,
+ fieldIdInLinkedTable: 'fldTitle',
+ isValid: true,
+ result: { type: 'number' },
+ },
+ },
+ { id: 'fldAuto', name: 'Auto', type: 'autoNumber' },
+ { id: 'fldDur', name: 'Duration', type: 'duration', options: { durationFormat: 'h:mm' } },
+ { id: 'fldBtn', name: 'Open', type: 'button' },
+ {
+ id: 'fldCreated',
+ name: 'Created at',
+ type: 'createdTime',
+ options: { result: { type: 'dateTime', options: { timeZone: 'Asia/Shanghai' } } },
+ },
+ ],
+ views: [
+ { id: 'viwGrid', name: 'All projects', type: 'grid' },
+ { id: 'viwTimeline', name: 'Roadmap', type: 'timeline' },
+ { id: 'viwForm', name: 'Intake', type: 'form' },
+ ],
+ },
+ {
+ id: 'tblTasks',
+ name: 'Tasks',
+ primaryFieldId: 'fldTitle',
+ fields: [
+ { id: 'fldTitle', name: 'Title', type: 'singleLineText' },
+ {
+ id: tasksLinkFieldId,
+ name: 'Project',
+ type: 'multipleRecordLinks',
+ options: {
+ linkedTableId: 'tblProjects',
+ prefersSingleRecordLink: true,
+ inverseLinkFieldId: projectsLinkFieldId,
+ isReversed: false,
+ },
+ },
+ {
+ id: 'fldProjName',
+ name: 'Project name',
+ type: 'multipleLookupValues',
+ options: {
+ recordLinkFieldId: tasksLinkFieldId,
+ fieldIdInLinkedTable: 'fldName',
+ isValid: true,
+ result: { type: 'singleLineText' },
+ },
+ },
+ {
+ id: 'fldBroken',
+ name: 'Broken formula',
+ type: 'formula',
+ options: { formula: '1/0', isValid: false, result: null },
+ },
+ {
+ id: 'fldAiSummary',
+ name: 'AI summary',
+ type: 'aiText',
+ options: {
+ prompt: ['Summarize ', { field: { fieldId: 'fldTitle' } }, ' briefly'],
+ referencedFieldIds: ['fldTitle'],
+ },
+ },
+ ],
+ views: [],
+ },
+];
+
+// isPrimary is not part of createFieldRoSchema; it is forwarded to v2 by the
+// internal create-table mapper (see table-open-api-v2.mapper.ts mapBaseField).
+const isPrimary = (ro: unknown) => (ro as { isPrimary?: boolean }).isPrimary;
+
+describe('buildAirtableImportPlan', () => {
+ const plan = buildAirtableImportPlan(buildSchema());
+ const projects = plan.tables[0];
+ const tasks = plan.tables[1];
+
+ it('produces phase-1 field ros that pass the create-field contract', () => {
+ for (const table of plan.tables) {
+ for (const field of table.fields) {
+ expect(
+ () => createFieldRoSchema.parse(field.ro),
+ `${table.name}.${field.ro.name}`
+ ).not.toThrow();
+ }
+ }
+ });
+
+ it('puts the primary field first with isPrimary set', () => {
+ expect(projects.fields[0].airtableFieldId).toBe('fldName');
+ expect(isPrimary(projects.fields[0].ro)).toBe(true);
+ expect(isPrimary(tasks.fields[0].ro)).toBe(true);
+ });
+
+ it('maps plain field types with options', () => {
+ const byAirtableId = new Map(projects.fields.map((field) => [field.airtableFieldId, field]));
+ expect(byAirtableId.get('fldNotes')?.ro.type).toBe(FieldType.LongText);
+ expect(byAirtableId.get('fldBudget')?.ro.options).toMatchObject({
+ formatting: { type: 'currency', precision: 2, symbol: '€' },
+ });
+ expect(byAirtableId.get('fldDue')?.ro.options).toMatchObject({
+ formatting: { date: 'D/M/YYYY', time: 'HH:mm', timeZone: 'UTC' },
+ });
+ expect(byAirtableId.get('fldOwner')?.ro.type).toBe(FieldType.User);
+ expect(byAirtableId.get('fldFiles')?.ro.type).toBe(FieldType.Attachment);
+ });
+
+ it('passes select choice colors through and falls back when missing', () => {
+ const priority = projects.fields.find((field) => field.airtableFieldId === 'fldPriority');
+ const choices = (priority?.ro.options as { choices: Array<{ name: string; color: string }> })
+ .choices;
+ expect(choices[0]).toMatchObject({ name: 'High', color: 'redBright' });
+ expect(choices[1].name).toBe('Low');
+ expect(choices[1].color).toBeTruthy();
+ // Airtable allows blank option names; Teable requires a non-empty name.
+ expect(choices[2].name.length).toBeGreaterThan(0);
+ });
+
+ it('owns a one-to-many link on the single-link side, independent of table order', () => {
+ // Projects.Tasks is multi and Tasks.Project is single; the single side must
+ // own the link so the relationship resolves to ManyOne (not ManyMany) no
+ // matter which table is traversed first.
+ expect(tasks.linkFields).toHaveLength(1);
+ expect(projects.linkFields).toHaveLength(0);
+ const link = tasks.linkFields[0];
+ expect(link.airtableFieldId).toBe(tasksLinkFieldId);
+ expect(link.prefersSingle).toBe(true);
+ expect(link.inverse).toMatchObject({
+ airtableFieldId: projectsLinkFieldId,
+ name: 'Tasks',
+ prefersSingle: false,
+ });
+ expect(plan.fieldIdMap[tasksLinkFieldId]).toBeTruthy();
+ expect(plan.fieldIdMap[projectsLinkFieldId]).toBeUndefined();
+ });
+
+ it('plans count as a rollup and lookup as a real lookup', () => {
+ expect(projects.countFields).toHaveLength(1);
+ expect(projects.countFields[0]).toMatchObject({
+ airtableLinkFieldId: projectsLinkFieldId,
+ airtableForeignTableId: 'tblTasks',
+ });
+ expect(tasks.lookupFields).toHaveLength(1);
+ expect(tasks.lookupFields[0]).toMatchObject({
+ airtableLinkFieldId: tasksLinkFieldId,
+ airtableForeignTableId: 'tblProjects',
+ airtableTargetFieldId: 'fldName',
+ });
+ });
+
+ it('carries a normalized AI prompt for aiText fields without degrading them upfront', () => {
+ const aiField = tasks.fields.find((field) => field.airtableFieldId === 'fldAiSummary');
+ expect(aiField?.ro.type).toBe(FieldType.LongText);
+ expect(aiField?.converter).toBe('aiText');
+ expect(aiField?.aiPromptParts).toEqual([
+ { text: 'Summarize ' },
+ { airtableFieldId: 'fldTitle', fieldName: 'Title' },
+ { text: ' briefly' },
+ ]);
+ expect(
+ plan.issues.some(
+ (issue) => issue.fieldName === 'AI summary' && issue.code === 'fieldDegraded'
+ )
+ ).toBe(false);
+ });
+
+ it('translates a compatible formula into a live Teable formula field', () => {
+ const score = projects.formulaFields.find((field) => field.airtableFieldId === 'fldScore');
+ expect(score?.expression).toBe('{fldBudget} * 2');
+ // A live formula is not duplicated as a snapshot, and is not reported as degraded.
+ expect(projects.fields.find((field) => field.airtableFieldId === 'fldScore')).toBeUndefined();
+ expect(plan.issues.some((issue) => issue.fieldName === 'Score')).toBe(false);
+ });
+
+ it('snapshots an invalid formula instead of emitting a broken live formula', () => {
+ expect(
+ tasks.formulaFields.find((field) => field.airtableFieldId === 'fldBroken')
+ ).toBeUndefined();
+ const snapshot = tasks.fields.find((field) => field.airtableFieldId === 'fldBroken');
+ expect(snapshot?.converter).toMatch(/^snapshot/);
+ expect(
+ plan.issues.some(
+ (issue) => issue.fieldName === 'Broken formula' && issue.code === 'fieldDegraded'
+ )
+ ).toBe(true);
+ });
+
+ it('snapshots a rollup when no shared base model supplies its aggregation', () => {
+ // the default `plan` is built without rollupSources
+ expect(projects.rollupFields).toHaveLength(0);
+ const snapshot = projects.fields.find((field) => field.airtableFieldId === 'fldTaskRollup');
+ expect(snapshot?.converter).toMatch(/^snapshot/);
+ });
+
+ it('recreates a rollup as a live Teable rollup when the shared base model supplies its aggregation', () => {
+ const rollupSources = new Map([
+ [
+ 'fldTaskRollup',
+ {
+ relationColumnId: projectsLinkFieldId,
+ foreignTableRollupColumnId: 'fldTitle',
+ aggregation: 'COUNTA(values)',
+ filter: null,
+ },
+ ],
+ ]);
+ const livePlan = buildAirtableImportPlan(buildSchema(), rollupSources);
+ const liveProjects = livePlan.tables[0];
+ const rollup = liveProjects.rollupFields.find((f) => f.airtableFieldId === 'fldTaskRollup');
+ expect(rollup?.expression).toBe('counta({values})');
+ expect(rollup?.airtableForeignTableId).toBe('tblTasks');
+ // a live rollup is not also emitted as a snapshot
+ expect(liveProjects.fields.find((f) => f.airtableFieldId === 'fldTaskRollup')).toBeUndefined();
+ });
+
+ it('degrades unsupported types and reports issues', () => {
+ const byAirtableId = new Map(projects.fields.map((field) => [field.airtableFieldId, field]));
+ expect(byAirtableId.get('fldAuto')?.ro.type).toBe(FieldType.Number);
+ expect(byAirtableId.get('fldDur')?.ro.type).toBe(FieldType.Number);
+ expect(byAirtableId.get('fldBtn')?.ro.type).toBe(FieldType.SingleLineText);
+ expect(byAirtableId.get('fldCreated')?.ro.type).toBe(FieldType.Date);
+ const degradedFields = plan.issues
+ .filter((issue) => issue.code === 'fieldDegraded')
+ .map((issue) => issue.fieldName);
+ expect(degradedFields).toEqual(
+ expect.arrayContaining(['Auto', 'Duration', 'Open', 'Created at'])
+ );
+ });
+
+ it('maps supported views and reports skipped ones, defaulting to a grid', () => {
+ expect(projects.views).toEqual([
+ { name: 'All projects', type: ViewType.Grid },
+ { name: 'Intake', type: ViewType.Form },
+ ]);
+ expect(
+ plan.issues.some((issue) => issue.code === 'viewSkipped' && issue.viewName === 'Roadmap')
+ ).toBe(true);
+ expect(tasks.views).toEqual([{ type: ViewType.Grid }]);
+ });
+
+ it('covers every non-inverse field in the field id map', () => {
+ const schema = buildSchema();
+ for (const table of schema) {
+ for (const field of table.fields) {
+ // Projects.Tasks (multi) is now the inverse side — the single-link
+ // Tasks.Project owns the link — so it is mapped at import time, not now.
+ if (field.id === projectsLinkFieldId) continue;
+ // formula/rollup are skipped (not imported), so they are not mapped.
+ if (field.type === 'formula' || field.type === 'rollup') continue;
+ expect(plan.fieldIdMap[field.id], `${table.name}.${field.name}`).toBeTruthy();
+ }
+ }
+ });
+});
+
+describe('buildAirtableImportPlan edge cases', () => {
+ it('degrades the primary field to a text snapshot when it maps to an incompatible type', () => {
+ const plan = buildAirtableImportPlan([
+ {
+ id: 'tblA',
+ name: 'A',
+ primaryFieldId: 'fldChk',
+ fields: [
+ { id: 'fldChk', name: 'Done', type: 'checkbox' },
+ { id: 'fldText', name: 'Text', type: 'singleLineText' },
+ ],
+ views: [],
+ },
+ ]);
+ const table = plan.tables[0];
+ expect(isPrimary(table.fields[0].ro)).toBe(true);
+ expect(table.fields[0].ro.type).toBe(FieldType.SingleLineText);
+ expect(table.fields[0].converter).toBe('snapshotText');
+ // the checkbox itself is replaced, not duplicated
+ expect(table.fields.filter((field) => field.airtableFieldId === 'fldChk')).toHaveLength(1);
+ });
+
+ it('treats a link without a valid inverse as one-way and self-links as pairable', () => {
+ const plan = buildAirtableImportPlan([
+ {
+ id: 'tblA',
+ name: 'A',
+ primaryFieldId: 'fldName',
+ fields: [
+ { id: 'fldName', name: 'Name', type: 'singleLineText' },
+ {
+ id: 'fldOneWay',
+ name: 'One way',
+ type: 'multipleRecordLinks',
+ options: { linkedTableId: 'tblA', prefersSingleRecordLink: false },
+ },
+ {
+ id: 'fldSelfA',
+ name: 'Parent',
+ type: 'multipleRecordLinks',
+ options: {
+ linkedTableId: 'tblA',
+ prefersSingleRecordLink: true,
+ inverseLinkFieldId: 'fldSelfB',
+ },
+ },
+ {
+ id: 'fldSelfB',
+ name: 'Children',
+ type: 'multipleRecordLinks',
+ options: {
+ linkedTableId: 'tblA',
+ prefersSingleRecordLink: false,
+ inverseLinkFieldId: 'fldSelfA',
+ },
+ },
+ ],
+ views: [],
+ },
+ ]);
+ const table = plan.tables[0];
+ expect(table.linkFields).toHaveLength(2);
+ const oneWay = table.linkFields.find((field) => field.airtableFieldId === 'fldOneWay');
+ expect(oneWay?.inverse).toBeUndefined();
+ const selfPair = table.linkFields.find((field) => field.airtableFieldId === 'fldSelfA');
+ expect(selfPair?.inverse?.airtableFieldId).toBe('fldSelfB');
+ });
+
+ it('carries a link "limit record selection to a view" into the plan', () => {
+ const plan = buildAirtableImportPlan([
+ {
+ id: 'tblA',
+ name: 'A',
+ primaryFieldId: 'fldName',
+ fields: [
+ { id: 'fldName', name: 'Name', type: 'singleLineText' },
+ {
+ id: 'fldLink',
+ name: 'Linked',
+ type: 'multipleRecordLinks',
+ options: {
+ linkedTableId: 'tblA',
+ prefersSingleRecordLink: false,
+ viewIdForRecordSelection: 'viwLimit',
+ },
+ },
+ ],
+ views: [],
+ },
+ ]);
+ const link = plan.tables[0].linkFields.find((field) => field.airtableFieldId === 'fldLink');
+ expect(link?.viewIdForRecordSelection).toBe('viwLimit');
+ });
+
+ it('falls back to a text snapshot for unknown future field types', () => {
+ const plan = buildAirtableImportPlan([
+ {
+ id: 'tblA',
+ name: 'A',
+ primaryFieldId: 'fldName',
+ fields: [
+ { id: 'fldName', name: 'Name', type: 'singleLineText' },
+ { id: 'fldNew', name: 'Mystery', type: 'someFutureType' },
+ ],
+ views: [],
+ },
+ ]);
+ const mystery = plan.tables[0].fields.find((field) => field.airtableFieldId === 'fldNew');
+ expect(mystery?.ro.type).toBe(FieldType.LongText);
+ expect(mystery?.converter).toBe('snapshotText');
+ });
+});
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-schema-mapper.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-schema-mapper.ts
new file mode 100644
index 0000000000..0da00070a2
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-schema-mapper.ts
@@ -0,0 +1,987 @@
+import type { IFieldRo, IViewRo } from '@teable/core';
+import {
+ Colors,
+ DateFormattingPreset,
+ FieldType,
+ generateFieldId,
+ NumberFormattingType,
+ RatingIcon,
+ SingleLineTextDisplayType,
+ TimeFormatting,
+ ViewType,
+} from '@teable/core';
+import type { IImportAirtableIssue } from '@teable/openapi';
+import { translateAirtableFormula } from './airtable-formula-translator';
+import { mapAirtableRollupAggregation } from './airtable-rollup-mapper';
+import type { IAirtableFilterGroup, IAirtableRollupSource } from './airtable-share.client';
+import type {
+ IAirtableField,
+ IAirtableFieldOptions,
+ IAirtableFieldResult,
+ IAirtableTable,
+} from './airtable.types';
+
+/** How raw Airtable cell values are converted before writing to Teable. */
+export type IAirtableCellConverter =
+ | 'string'
+ | 'number'
+ | 'boolean'
+ | 'dateTime'
+ | 'stringArray'
+ | 'user'
+ | 'attachment'
+ | 'barcode'
+ | 'button'
+ | 'aiText'
+ | 'collaboratorText'
+ | 'snapshotText'
+ | 'snapshotNumber'
+ | 'snapshotDate'
+ | 'snapshotCheckbox';
+
+/** A normalized chunk of an Airtable AI field prompt: plain text or a field reference. */
+export interface IAiPromptPart {
+ text?: string;
+ airtableFieldId?: string;
+ fieldName?: string;
+}
+
+export interface IPlannedDirectField {
+ airtableFieldId: string;
+ converter: IAirtableCellConverter;
+ ro: IFieldRo;
+ /**
+ * Present for Airtable aiText fields. The importer turns it into a Teable
+ * AI field config when the target base has an AI model configured.
+ */
+ aiPromptParts?: IAiPromptPart[];
+}
+
+export interface IPlannedLinkField {
+ airtableFieldId: string;
+ teableFieldId: string;
+ name: string;
+ description?: string;
+ airtableForeignTableId: string;
+ prefersSingle: boolean;
+ /** The paired two-way field in the foreign table, mapped to the auto-created symmetric field. */
+ inverse?: { airtableFieldId: string; name: string; prefersSingle: boolean };
+ /** Airtable's "limit record selection to a view" — a view id in the foreign table. */
+ viewIdForRecordSelection?: string;
+}
+
+export interface IPlannedLookupField {
+ airtableFieldId: string;
+ teableFieldId: string;
+ name: string;
+ description?: string;
+ airtableLinkFieldId: string;
+ airtableForeignTableId: string;
+ airtableTargetFieldId: string;
+}
+
+export interface IPlannedCountField {
+ airtableFieldId: string;
+ teableFieldId: string;
+ name: string;
+ description?: string;
+ airtableLinkFieldId: string;
+ airtableForeignTableId: string;
+}
+
+export interface IPlannedFormulaField {
+ airtableFieldId: string;
+ teableFieldId: string;
+ name: string;
+ description?: string;
+ /** Translated Teable expression, still referencing fields by Airtable id (`{fldXXX}`). */
+ expression: string;
+}
+
+export interface IPlannedRollupField {
+ airtableFieldId: string;
+ teableFieldId: string;
+ name: string;
+ description?: string;
+ /** Teable rollup expression, e.g. `sum({values})`. */
+ expression: string;
+ airtableLinkFieldId: string;
+ airtableForeignTableId: string;
+ airtableForeignFieldId: string;
+ /** Raw Airtable "only include records that meet conditions" filter, mapped at creation. */
+ filter: IAirtableFilterGroup | null;
+}
+
+/** The Airtable view each created Teable view came from (for view-config import). */
+export interface IPlannedViewSource {
+ airtableViewId: string;
+ teableViewType: ViewType;
+}
+
+export interface IAirtableTablePlan {
+ airtableTableId: string;
+ name: string;
+ description?: string;
+ /** Phase-1 fields in creation order, primary first. */
+ fields: IPlannedDirectField[];
+ views: IViewRo[];
+ /** Aligned index-wise with `views`; empty entries (default fallback) have no source. */
+ viewSources: IPlannedViewSource[];
+ /** Phase-2 fields, created after all tables and records exist. */
+ linkFields: IPlannedLinkField[];
+ lookupFields: IPlannedLookupField[];
+ countFields: IPlannedCountField[];
+ /** Phase-3 fields, created last so every referenced field id is already mapped. */
+ formulaFields: IPlannedFormulaField[];
+ /** Live rollups recreated from the shared base model, created with lookups/counts. */
+ rollupFields: IPlannedRollupField[];
+}
+
+export interface IAirtableImportPlan {
+ tables: IAirtableTablePlan[];
+ /**
+ * airtable field id -> pre-generated teable field id. Table ids cannot be
+ * client-specified, the importer records them from the create-table
+ * responses. Inverse link fields are mapped during import as well (to the
+ * auto-created symmetric fields).
+ */
+ fieldIdMap: Record;
+ issues: IImportAirtableIssue[];
+}
+
+const maxNumberPrecision = 5;
+const maxDescriptionLength = 1000;
+const longTextSnapshotLabel = 'longText snapshot';
+
+const teableColorValues = new Set(Object.values(Colors));
+
+/** Rotating palette for select choices when Airtable has colored options disabled. */
+const defaultChoiceColors = [
+ Colors.BlueLight1,
+ Colors.GreenLight1,
+ Colors.YellowLight1,
+ Colors.OrangeLight1,
+ Colors.RedLight1,
+ Colors.PurpleLight1,
+ Colors.PinkLight1,
+ Colors.CyanLight1,
+ Colors.TealLight1,
+ Colors.GrayLight1,
+];
+
+const viewTypeMap: Record = {
+ grid: ViewType.Grid,
+ form: ViewType.Form,
+ calendar: ViewType.Calendar,
+ gallery: ViewType.Gallery,
+ kanban: ViewType.Kanban,
+};
+
+const ratingIconMap: Record = {
+ star: RatingIcon.Star,
+ heart: RatingIcon.Heart,
+ thumbsUp: RatingIcon.ThumbUp,
+ flag: RatingIcon.Flame,
+ dot: RatingIcon.Moon,
+};
+
+const primaryIncompatibleTypes = new Set([FieldType.Attachment, FieldType.Checkbox]);
+
+const clampPrecision = (precision: unknown) => {
+ const value = typeof precision === 'number' ? Math.round(precision) : 0;
+ return Math.min(Math.max(value, 0), maxNumberPrecision);
+};
+
+const mapTimeZone = (timeZone: string | undefined) => {
+ if (!timeZone || timeZone === 'client' || timeZone === 'utc') {
+ return 'UTC';
+ }
+ return timeZone;
+};
+
+const mapDateFormat = (options: IAirtableFieldOptions | undefined) => {
+ switch (options?.dateFormat?.name) {
+ case 'us':
+ return DateFormattingPreset.US;
+ case 'european':
+ return DateFormattingPreset.European;
+ default:
+ return DateFormattingPreset.ISO;
+ }
+};
+
+const mapTimeFormat = (options: IAirtableFieldOptions | undefined) => {
+ switch (options?.timeFormat?.name) {
+ case '12hour':
+ return TimeFormatting.Hour12;
+ case '24hour':
+ return TimeFormatting.Hour24;
+ default:
+ return TimeFormatting.None;
+ }
+};
+
+const buildDatetimeFormatting = (
+ options: IAirtableFieldOptions | undefined,
+ withTime: boolean
+) => ({
+ date: mapDateFormat(options),
+ time: withTime ? mapTimeFormat(options) : TimeFormatting.None,
+ timeZone: mapTimeZone(options?.timeZone),
+});
+
+const mapRatingColor = (color: string | undefined) => {
+ if (color === Colors.YellowBright || color === 'orangeBright') {
+ return Colors.YellowBright;
+ }
+ if (color === Colors.RedBright || color === 'pinkBright' || color === 'purpleBright') {
+ return Colors.RedBright;
+ }
+ return Colors.TealBright;
+};
+
+const mapSelectChoices = (options: IAirtableFieldOptions | undefined) => {
+ const choices = options?.choices ?? [];
+ return choices.map((choice, index) => ({
+ // Airtable allows blank option names; Teable requires at least one char.
+ name: choice.name?.trim() ? choice.name : `(blank ${index + 1})`,
+ color: (teableColorValues.has(choice.color ?? '')
+ ? choice.color
+ : defaultChoiceColors[index % defaultChoiceColors.length]) as Colors,
+ }));
+};
+
+const truncateDescription = (description: string) =>
+ description.length > maxDescriptionLength
+ ? `${description.slice(0, maxDescriptionLength - 1)}…`
+ : description;
+
+/** Rewrites `{fldXXX}` references inside a formula to readable `{Field Name}` references. */
+const humanizeFormula = (formula: string, fieldNameById: Map) =>
+ formula.replace(/\{(fld[a-zA-Z0-9]+)\}/g, (match, fieldId: string) => {
+ const name = fieldNameById.get(fieldId);
+ return name ? `{${name}}` : match;
+ });
+
+interface IDirectMapping {
+ kind: 'direct';
+ type: FieldType;
+ options?: IFieldRo['options'];
+ converter: IAirtableCellConverter;
+ degradedTo?: string;
+ aiPromptParts?: IAiPromptPart[];
+}
+
+/**
+ * Normalizes an Airtable aiText prompt (an array of text chunks and field
+ * reference objects) so the importer can rebuild it as a Teable AI prompt.
+ */
+const normalizeAiPromptParts = (
+ prompt: unknown,
+ fieldNameById: Map
+): IAiPromptPart[] => {
+ if (!Array.isArray(prompt)) return [];
+ const parts: IAiPromptPart[] = [];
+ for (const part of prompt) {
+ if (typeof part === 'string') {
+ parts.push({ text: part });
+ continue;
+ }
+ if (part && typeof part === 'object') {
+ const record = part as { field?: { fieldId?: unknown }; fieldId?: unknown };
+ const fieldId = record.field?.fieldId ?? record.fieldId;
+ if (typeof fieldId === 'string') {
+ parts.push({ airtableFieldId: fieldId, fieldName: fieldNameById.get(fieldId) });
+ }
+ }
+ }
+ return parts;
+};
+
+interface ISkipMapping {
+ kind: 'skip';
+ reason: string;
+}
+
+type IFieldMapping = IDirectMapping | ISkipMapping;
+
+const textMapping = (showAs?: SingleLineTextDisplayType): IDirectMapping => ({
+ kind: 'direct',
+ type: FieldType.SingleLineText,
+ options: showAs ? { showAs: { type: showAs } } : {},
+ converter: 'string',
+});
+
+const numberMapping = (
+ formattingType: NumberFormattingType,
+ options: IAirtableFieldOptions | undefined,
+ converter: IAirtableCellConverter = 'number'
+): IDirectMapping => ({
+ kind: 'direct',
+ type: FieldType.Number,
+ options: {
+ formatting: {
+ type: formattingType,
+ precision: clampPrecision(options?.precision),
+ ...(formattingType === NumberFormattingType.Currency
+ ? { symbol: options?.symbol ?? '$' }
+ : {}),
+ },
+ },
+ converter,
+});
+
+/**
+ * Maps the `result` config of a computed Airtable field (formula/rollup/lookup)
+ * to a plain Teable field that stores a static snapshot of the computed values.
+ */
+const snapshotMappingFromResult = (
+ result: IAirtableFieldResult | null | undefined
+): IDirectMapping => {
+ const options = result?.options as IAirtableFieldOptions | undefined;
+ switch (result?.type) {
+ case 'number':
+ return numberMapping(NumberFormattingType.Decimal, options, 'snapshotNumber');
+ case 'percent':
+ return numberMapping(NumberFormattingType.Percent, options, 'snapshotNumber');
+ case 'currency':
+ return numberMapping(NumberFormattingType.Currency, options, 'snapshotNumber');
+ case 'duration':
+ case 'rating':
+ case 'autoNumber':
+ case 'count':
+ return {
+ kind: 'direct',
+ type: FieldType.Number,
+ options: {
+ formatting: { type: NumberFormattingType.Decimal, precision: clampPrecision(0) },
+ },
+ converter: 'snapshotNumber',
+ };
+ case 'date':
+ case 'dateTime':
+ case 'createdTime':
+ case 'lastModifiedTime':
+ return {
+ kind: 'direct',
+ type: FieldType.Date,
+ options: { formatting: buildDatetimeFormatting(options, result.type !== 'date') },
+ converter: 'snapshotDate',
+ };
+ case 'checkbox':
+ return {
+ kind: 'direct',
+ type: FieldType.Checkbox,
+ options: {},
+ converter: 'snapshotCheckbox',
+ };
+ default:
+ return {
+ kind: 'direct',
+ type: FieldType.SingleLineText,
+ options: {},
+ converter: 'snapshotText',
+ };
+ }
+};
+
+// eslint-disable-next-line sonarjs/cognitive-complexity
+const mapField = (field: IAirtableField, fieldNameById: Map): IFieldMapping => {
+ const { type, options } = field;
+ switch (type) {
+ case 'singleLineText':
+ return textMapping();
+ case 'email':
+ return textMapping(SingleLineTextDisplayType.Email);
+ case 'url':
+ return textMapping(SingleLineTextDisplayType.Url);
+ case 'phoneNumber':
+ return textMapping(SingleLineTextDisplayType.Phone);
+ case 'multilineText':
+ case 'richText':
+ return { kind: 'direct', type: FieldType.LongText, options: {}, converter: 'string' };
+ case 'number':
+ return numberMapping(NumberFormattingType.Decimal, options);
+ case 'percent':
+ return numberMapping(NumberFormattingType.Percent, options);
+ case 'currency':
+ return numberMapping(NumberFormattingType.Currency, options);
+ case 'duration':
+ case 'autoNumber':
+ // Teable has no duration type (Airtable stores duration as seconds) and
+ // cannot back-fill auto numbers — keep both as plain numbers.
+ return {
+ ...numberMapping(NumberFormattingType.Decimal, { precision: 0 }),
+ degradedTo: 'number',
+ };
+ case 'rating':
+ return {
+ kind: 'direct',
+ type: FieldType.Rating,
+ options: {
+ icon: ratingIconMap[options?.icon ?? ''] ?? RatingIcon.Star,
+ color: mapRatingColor(options?.color),
+ max: Math.min(Math.max(options?.max ?? 5, 1), 10),
+ },
+ converter: 'number',
+ };
+ case 'checkbox':
+ return { kind: 'direct', type: FieldType.Checkbox, options: {}, converter: 'boolean' };
+ case 'singleSelect':
+ return {
+ kind: 'direct',
+ type: FieldType.SingleSelect,
+ options: { choices: mapSelectChoices(options) },
+ converter: 'string',
+ };
+ case 'multipleSelects':
+ return {
+ kind: 'direct',
+ type: FieldType.MultipleSelect,
+ options: { choices: mapSelectChoices(options) },
+ converter: 'stringArray',
+ };
+ case 'externalSyncSource':
+ return {
+ kind: 'direct',
+ type: FieldType.SingleSelect,
+ options: { choices: mapSelectChoices(options) },
+ converter: 'string',
+ degradedTo: 'singleSelect',
+ };
+ case 'date':
+ return {
+ kind: 'direct',
+ type: FieldType.Date,
+ options: { formatting: buildDatetimeFormatting(options, false) },
+ converter: 'dateTime',
+ };
+ case 'dateTime':
+ return {
+ kind: 'direct',
+ type: FieldType.Date,
+ options: { formatting: buildDatetimeFormatting(options, true) },
+ converter: 'dateTime',
+ };
+ case 'createdTime':
+ case 'lastModifiedTime': {
+ // Teable would recompute these to the import time; keep the original
+ // values as a static date snapshot instead.
+ const result = options?.result;
+ const resultOptions = result?.options as IAirtableFieldOptions | undefined;
+ return {
+ kind: 'direct',
+ type: FieldType.Date,
+ options: { formatting: buildDatetimeFormatting(resultOptions, true) },
+ converter: 'dateTime',
+ degradedTo: 'date snapshot',
+ };
+ }
+ case 'createdBy':
+ case 'lastModifiedBy':
+ return {
+ kind: 'direct',
+ type: FieldType.SingleLineText,
+ options: {},
+ converter: 'collaboratorText',
+ degradedTo: 'singleLineText snapshot',
+ };
+ case 'singleCollaborator':
+ return { kind: 'direct', type: FieldType.User, options: {}, converter: 'user' };
+ case 'multipleCollaborators':
+ return {
+ kind: 'direct',
+ type: FieldType.User,
+ options: { isMultiple: true },
+ converter: 'user',
+ };
+ case 'multipleAttachments':
+ return { kind: 'direct', type: FieldType.Attachment, options: {}, converter: 'attachment' };
+ case 'barcode':
+ return {
+ kind: 'direct',
+ type: FieldType.SingleLineText,
+ options: {},
+ converter: 'barcode',
+ degradedTo: 'singleLineText',
+ };
+ case 'button':
+ return {
+ kind: 'direct',
+ type: FieldType.SingleLineText,
+ options: {},
+ converter: 'button',
+ degradedTo: 'singleLineText',
+ };
+ case 'aiText':
+ // Becomes a Teable AI field when the target base has an AI model
+ // configured; otherwise the importer reports it as a snapshot.
+ return {
+ kind: 'direct',
+ type: FieldType.LongText,
+ options: {},
+ converter: 'aiText',
+ aiPromptParts: normalizeAiPromptParts(options?.prompt, fieldNameById),
+ };
+ // formula and rollup are handled in the main loop (translate-or-snapshot).
+ default:
+ // Unknown / future Airtable field types degrade to a text snapshot so
+ // the import never fails on them.
+ return {
+ kind: 'direct',
+ type: FieldType.LongText,
+ options: {},
+ converter: 'snapshotText',
+ degradedTo: longTextSnapshotLabel,
+ };
+ }
+};
+
+const buildComputedDescription = (
+ field: IAirtableField,
+ fieldNameById: Map
+): string | undefined => {
+ if (field.type === 'formula' && field.options?.formula) {
+ const formula = humanizeFormula(field.options.formula, fieldNameById);
+ return truncateDescription(`Airtable formula: ${formula}`);
+ }
+ if (field.type === 'rollup') {
+ const linkName = fieldNameById.get(field.options?.recordLinkFieldId ?? '');
+ const targetName = fieldNameById.get(field.options?.fieldIdInLinkedTable ?? '');
+ if (linkName && targetName) {
+ return truncateDescription(`Airtable rollup of "${targetName}" via "${linkName}"`);
+ }
+ }
+ if (field.type === 'multipleLookupValues') {
+ const linkName = fieldNameById.get(field.options?.recordLinkFieldId ?? '');
+ const targetName = fieldNameById.get(field.options?.fieldIdInLinkedTable ?? '');
+ if (linkName && targetName) {
+ return truncateDescription(`Airtable lookup of "${targetName}" via "${linkName}"`);
+ }
+ }
+ return field.description ? truncateDescription(field.description) : undefined;
+};
+
+const mapViews = (
+ table: IAirtableTable,
+ issues: IImportAirtableIssue[]
+): { views: IViewRo[]; sources: IPlannedViewSource[] } => {
+ const views: IViewRo[] = [];
+ const sources: IPlannedViewSource[] = [];
+ for (const view of table.views) {
+ const viewType = viewTypeMap[view.type];
+ if (!viewType) {
+ issues.push({
+ code: 'viewSkipped',
+ tableName: table.name,
+ viewName: view.name,
+ fromType: view.type,
+ });
+ continue;
+ }
+ views.push({ name: view.name, type: viewType } as IViewRo);
+ sources.push({ airtableViewId: view.id, teableViewType: viewType });
+ }
+ if (views.length === 0) {
+ views.push({ type: ViewType.Grid } as IViewRo);
+ }
+ return { views, sources };
+};
+
+/**
+ * Builds the full import plan for an Airtable base schema: phase-1 plain
+ * fields (created together with the tables), view shells, and phase-2
+ * link/lookup/count fields (created once all tables and records exist).
+ */
+/* eslint-disable sonarjs/cognitive-complexity -- one large switch over Airtable field types */
+export const buildAirtableImportPlan = (
+ tables: IAirtableTable[],
+ /** Rollup sources from the shared base model; absent when no share link was given. */
+ rollupSources?: Map
+): IAirtableImportPlan => {
+ const issues: IImportAirtableIssue[] = [];
+ const fieldIdMap: Record = {};
+ const tableById = new Map(tables.map((table) => [table.id, table]));
+ const fieldNameById = new Map();
+ for (const table of tables) {
+ for (const field of table.fields) {
+ fieldNameById.set(field.id, field.name);
+ }
+ }
+
+ // Two-way links appear as a field on both tables; the second one of each
+ // pair is realized by renaming the symmetric field Teable auto-creates.
+ const inverseFieldIds = new Set();
+ for (const table of tables) {
+ for (const field of table.fields) {
+ if (field.type !== 'multipleRecordLinks' || inverseFieldIds.has(field.id)) {
+ continue;
+ }
+ const inverseId = field.options?.inverseLinkFieldId;
+ const foreignTable = tableById.get(field.options?.linkedTableId ?? '');
+ const inverseField = foreignTable?.fields.find(
+ (candidate) => candidate.id === inverseId && candidate.type === 'multipleRecordLinks'
+ );
+ if (inverseField && inverseField.id !== field.id) {
+ // Own the link from the single-link side so a one-to-many keeps its
+ // ManyOne cardinality regardless of table/field traversal order. Only
+ // flips when exactly one side prefers a single link; ties (both single
+ // or both multi) keep traversal order.
+ const fieldSingle = field.options?.prefersSingleRecordLink === true;
+ const inverseSingle = inverseField.options?.prefersSingleRecordLink === true;
+ const inverseOwns = inverseSingle && !fieldSingle;
+ inverseFieldIds.add(inverseOwns ? field.id : inverseField.id);
+ }
+ }
+ }
+
+ const plannedLinkIds = new Set();
+ for (const table of tables) {
+ for (const field of table.fields) {
+ if (
+ field.type === 'multipleRecordLinks' &&
+ tableById.has(field.options?.linkedTableId ?? '') &&
+ !inverseFieldIds.has(field.id)
+ ) {
+ plannedLinkIds.add(field.id);
+ }
+ }
+ }
+
+ // eslint-disable-next-line sonarjs/cognitive-complexity
+ const tablePlans: IAirtableTablePlan[] = tables.map((table) => {
+ const { views, sources } = mapViews(table, issues);
+ const plan: IAirtableTablePlan = {
+ airtableTableId: table.id,
+ name: table.name,
+ description: table.description,
+ fields: [],
+ views,
+ viewSources: sources,
+ linkFields: [],
+ lookupFields: [],
+ countFields: [],
+ formulaFields: [],
+ rollupFields: [],
+ };
+
+ for (const field of table.fields) {
+ const description = buildComputedDescription(field, fieldNameById);
+
+ if (field.type === 'multipleRecordLinks') {
+ if (inverseFieldIds.has(field.id)) {
+ // Realized as the symmetric side of the owning link field.
+ continue;
+ }
+ const foreignTable = tableById.get(field.options?.linkedTableId ?? '');
+ if (!foreignTable) {
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: table.name,
+ fieldName: field.name,
+ fromType: field.type,
+ reason: 'linked table not found',
+ });
+ continue;
+ }
+ const inverseId = field.options?.inverseLinkFieldId;
+ const inverseField = foreignTable.fields.find(
+ (candidate) => candidate.id === inverseId && inverseFieldIds.has(candidate.id)
+ );
+ const teableFieldId = generateFieldId();
+ fieldIdMap[field.id] = teableFieldId;
+ plan.linkFields.push({
+ airtableFieldId: field.id,
+ teableFieldId,
+ name: field.name,
+ description,
+ airtableForeignTableId: foreignTable.id,
+ prefersSingle: field.options?.prefersSingleRecordLink === true,
+ viewIdForRecordSelection: field.options?.viewIdForRecordSelection,
+ inverse: inverseField
+ ? {
+ airtableFieldId: inverseField.id,
+ name: inverseField.name,
+ prefersSingle: inverseField.options?.prefersSingleRecordLink === true,
+ }
+ : undefined,
+ });
+ continue;
+ }
+
+ if (field.type === 'count') {
+ const linkFieldId = field.options?.recordLinkFieldId ?? '';
+ const linkField = table.fields.find((candidate) => candidate.id === linkFieldId);
+ const foreignTableId = linkField?.options?.linkedTableId ?? '';
+ if (
+ field.options?.isValid !== false &&
+ (plannedLinkIds.has(linkFieldId) || inverseFieldIds.has(linkFieldId)) &&
+ tableById.has(foreignTableId)
+ ) {
+ const teableFieldId = generateFieldId();
+ fieldIdMap[field.id] = teableFieldId;
+ plan.countFields.push({
+ airtableFieldId: field.id,
+ teableFieldId,
+ name: field.name,
+ description,
+ airtableLinkFieldId: linkFieldId,
+ airtableForeignTableId: foreignTableId,
+ });
+ continue;
+ }
+ const mapping = snapshotMappingFromResult({ type: 'count' });
+ issues.push({
+ code: 'fieldDegraded',
+ tableName: table.name,
+ fieldName: field.name,
+ fromType: field.type,
+ toType: 'number snapshot',
+ });
+ const teableFieldId = generateFieldId();
+ fieldIdMap[field.id] = teableFieldId;
+ plan.fields.push({
+ airtableFieldId: field.id,
+ converter: mapping.converter,
+ ro: {
+ id: teableFieldId,
+ name: field.name,
+ description,
+ type: mapping.type,
+ options: mapping.options,
+ } as IFieldRo,
+ });
+ continue;
+ }
+
+ if (field.type === 'multipleLookupValues') {
+ const linkFieldId = field.options?.recordLinkFieldId ?? '';
+ const targetFieldId = field.options?.fieldIdInLinkedTable ?? '';
+ const linkField = table.fields.find((candidate) => candidate.id === linkFieldId);
+ const foreignTable = tableById.get(linkField?.options?.linkedTableId ?? '');
+ const targetField = foreignTable?.fields.find(
+ (candidate) => candidate.id === targetFieldId
+ );
+ const targetIsPlain =
+ targetField &&
+ !['multipleRecordLinks', 'multipleLookupValues', 'rollup', 'count'].includes(
+ targetField.type
+ );
+ if (
+ field.options?.isValid !== false &&
+ (plannedLinkIds.has(linkFieldId) || inverseFieldIds.has(linkFieldId)) &&
+ foreignTable &&
+ targetIsPlain
+ ) {
+ const teableFieldId = generateFieldId();
+ fieldIdMap[field.id] = teableFieldId;
+ plan.lookupFields.push({
+ airtableFieldId: field.id,
+ teableFieldId,
+ name: field.name,
+ description,
+ airtableLinkFieldId: linkFieldId,
+ airtableForeignTableId: foreignTable.id,
+ airtableTargetFieldId: targetFieldId,
+ });
+ continue;
+ }
+ issues.push({
+ code: 'fieldDegraded',
+ tableName: table.name,
+ fieldName: field.name,
+ fromType: field.type,
+ toType: longTextSnapshotLabel,
+ });
+ const teableFieldId = generateFieldId();
+ fieldIdMap[field.id] = teableFieldId;
+ plan.fields.push({
+ airtableFieldId: field.id,
+ converter: 'snapshotText',
+ ro: {
+ id: teableFieldId,
+ name: field.name,
+ description,
+ type: FieldType.LongText,
+ options: {},
+ } as IFieldRo,
+ });
+ continue;
+ }
+
+ if (field.type === 'formula' || field.type === 'rollup') {
+ // Recreate as a live computed field when possible: a formula when every
+ // function translates, a rollup when its aggregation maps and the base
+ // model (from the shared link) supplied its source. Otherwise the
+ // computed values are kept as a typed static snapshot so no data is lost.
+ const translation =
+ field.type === 'formula' && field.options?.formula && field.options?.isValid !== false
+ ? translateAirtableFormula(field.options.formula)
+ : null;
+ const teableFieldId = generateFieldId();
+ fieldIdMap[field.id] = teableFieldId;
+ if (translation?.ok) {
+ plan.formulaFields.push({
+ airtableFieldId: field.id,
+ teableFieldId,
+ name: field.name,
+ description,
+ expression: translation.expression,
+ });
+ continue;
+ }
+ if (field.type === 'rollup') {
+ const source = rollupSources?.get(field.id);
+ const expression = source ? mapAirtableRollupAggregation(source.aggregation) : null;
+ const linkField = table.fields.find(
+ (candidate) => candidate.id === source?.relationColumnId
+ );
+ const foreignTableId = linkField?.options?.linkedTableId ?? '';
+ const linkIsImported =
+ !!source &&
+ (plannedLinkIds.has(source.relationColumnId) ||
+ inverseFieldIds.has(source.relationColumnId));
+ if (source && expression && linkIsImported && tableById.has(foreignTableId)) {
+ plan.rollupFields.push({
+ airtableFieldId: field.id,
+ teableFieldId,
+ name: field.name,
+ description,
+ expression,
+ airtableLinkFieldId: source.relationColumnId,
+ airtableForeignTableId: foreignTableId,
+ airtableForeignFieldId: source.foreignTableRollupColumnId,
+ filter: source.filter,
+ });
+ continue;
+ }
+ }
+ const mapping = snapshotMappingFromResult(field.options?.result);
+ issues.push({
+ code: 'fieldDegraded',
+ tableName: table.name,
+ fieldName: field.name,
+ fromType: field.type,
+ toType: `${mapping.type} snapshot`,
+ });
+ plan.fields.push({
+ airtableFieldId: field.id,
+ converter: mapping.converter,
+ ro: {
+ id: teableFieldId,
+ name: field.name,
+ description,
+ type: mapping.type,
+ options: mapping.options,
+ } as IFieldRo,
+ });
+ continue;
+ }
+
+ const mapping = mapField(field, fieldNameById);
+ if (mapping.kind === 'skip') {
+ issues.push({
+ code: 'fieldSkipped',
+ tableName: table.name,
+ fieldName: field.name,
+ fromType: field.type,
+ reason: mapping.reason,
+ });
+ continue;
+ }
+ if (mapping.degradedTo) {
+ issues.push({
+ code: 'fieldDegraded',
+ tableName: table.name,
+ fieldName: field.name,
+ fromType: field.type,
+ toType: mapping.degradedTo,
+ });
+ }
+ const teableFieldId = generateFieldId();
+ fieldIdMap[field.id] = teableFieldId;
+ plan.fields.push({
+ airtableFieldId: field.id,
+ converter: mapping.converter,
+ aiPromptParts: mapping.aiPromptParts,
+ ro: {
+ id: teableFieldId,
+ name: field.name,
+ description,
+ type: mapping.type,
+ options: mapping.options,
+ } as IFieldRo,
+ });
+ }
+
+ applyPrimaryField(table, plan, issues, fieldIdMap);
+ return plan;
+ });
+
+ return { tables: tablePlans, fieldIdMap, issues };
+};
+/* eslint-enable sonarjs/cognitive-complexity */
+
+/**
+ * Ensures the table has a valid primary field as its first phase-1 field.
+ * When the Airtable primary field maps to something that cannot be a Teable
+ * primary (link, attachment, checkbox…), it is replaced by a text snapshot.
+ */
+const applyPrimaryField = (
+ table: IAirtableTable,
+ plan: IAirtableTablePlan,
+ issues: IImportAirtableIssue[],
+ fieldIdMap: Record
+) => {
+ const primaryIndex = plan.fields.findIndex(
+ (field) => field.airtableFieldId === table.primaryFieldId
+ );
+ const primaryField = primaryIndex >= 0 ? plan.fields[primaryIndex] : undefined;
+ const isCompatible = primaryField && !primaryIncompatibleTypes.has(primaryField.ro.type);
+
+ if (primaryField && isCompatible) {
+ plan.fields.splice(primaryIndex, 1);
+ plan.fields.unshift({
+ ...primaryField,
+ ro: { ...primaryField.ro, isPrimary: true } as IFieldRo,
+ });
+ return;
+ }
+
+ // The Airtable primary maps to a phase-2 field or an incompatible type:
+ // degrade it (or synthesize one) as a text snapshot primary.
+ const airtableField = table.fields.find((field) => field.id === table.primaryFieldId);
+ const name = airtableField?.name ?? 'Name';
+ if (primaryField) {
+ plan.fields.splice(primaryIndex, 1);
+ }
+ plan.linkFields = plan.linkFields.filter(
+ (field) => field.airtableFieldId !== table.primaryFieldId
+ );
+ plan.lookupFields = plan.lookupFields.filter(
+ (field) => field.airtableFieldId !== table.primaryFieldId
+ );
+ plan.countFields = plan.countFields.filter(
+ (field) => field.airtableFieldId !== table.primaryFieldId
+ );
+ const teableFieldId = generateFieldId();
+ fieldIdMap[table.primaryFieldId] = teableFieldId;
+ issues.push({
+ code: 'fieldDegraded',
+ tableName: table.name,
+ fieldName: name,
+ fromType: airtableField?.type ?? 'unknown',
+ toType: 'singleLineText snapshot',
+ });
+ plan.fields.unshift({
+ airtableFieldId: table.primaryFieldId,
+ converter: 'snapshotText',
+ ro: {
+ id: teableFieldId,
+ name,
+ type: FieldType.SingleLineText,
+ options: {},
+ isPrimary: true,
+ } as IFieldRo,
+ });
+};
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-share.client.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-share.client.ts
new file mode 100644
index 0000000000..f1e96e4ad8
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-share.client.ts
@@ -0,0 +1,382 @@
+import { Readable } from 'node:stream';
+import { finished } from 'node:stream/promises';
+import { parser } from 'stream-json';
+import Assembler from 'stream-json/Assembler';
+import { ignore } from 'stream-json/filters/Ignore';
+
+/**
+ * Reads Airtable view configuration (filters/sorts/groups/kanban stacking) that
+ * the official Web API does not expose, through the same undocumented endpoint
+ * behind a public shared-base link that Baserow and NocoDB use. It is read-only,
+ * authenticated solely by the signed access policy embedded in the share page,
+ * and used as an opt-in enhancement that always degrades gracefully.
+ */
+const shareBaseUrl = 'https://airtable.com';
+const shareApiBaseUrl = 'https://airtable.com/v0.3';
+const browserUserAgent =
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
+ '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+// Polite spacing between sequential reads of the undocumented private endpoint.
+const minRequestIntervalMs = 200;
+const sessionNotResolvedMessage = 'Share session not resolved.';
+
+export type IAirtableShareErrorReason =
+ | 'not_public'
+ | 'not_a_base'
+ | 'requires_auth'
+ | 'base_mismatch'
+ | 'resolve_failed';
+
+export class AirtableShareError extends Error {
+ constructor(
+ message: string,
+ public readonly reason: IAirtableShareErrorReason
+ ) {
+ super(message);
+ this.name = 'AirtableShareError';
+ }
+}
+
+/** Opaque, time-boxed session scraped from the share page; replayed verbatim. */
+export interface IAirtableShareSession {
+ appId: string;
+ accessPolicy: string;
+ requestId: string;
+ pageLoadId: string;
+ codeVersion: string;
+ cookie: string;
+}
+
+export interface IAirtableFilterLeaf {
+ columnId: string;
+ operator: string;
+ value: unknown;
+}
+
+export interface IAirtableFilterGroup {
+ conjunction: 'and' | 'or';
+ filterSet: Array;
+}
+
+export interface IAirtableSort {
+ columnId: string;
+ ascending: boolean;
+}
+
+export interface IAirtableGroupLevel {
+ columnId: string;
+ order: 'ascending' | 'descending';
+}
+
+export interface IAirtableViewConfig {
+ filters: IAirtableFilterGroup | null;
+ sorts: IAirtableSort[] | null;
+ groupLevels: IAirtableGroupLevel[] | null;
+ metadata: Record | undefined;
+}
+
+/** A rollup column's source, read from the base model the official API does not expose. */
+export interface IAirtableRollupSource {
+ /** Airtable link field the rollup summarizes through. */
+ relationColumnId: string;
+ /** Airtable field, in the linked table, being rolled up. */
+ foreignTableRollupColumnId: string;
+ /** Aggregation formula, e.g. "SUM(values)". */
+ aggregation: string;
+ /** Optional "only include linked records that meet conditions" filter. */
+ filter: IAirtableFilterGroup | null;
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Locates a rollup's record-selection filter inside its typeOptions by shape
+ * (an object carrying a `filterSet`), so it is found regardless of the exact key.
+ */
+const findAirtableFilter = (typeOptions: Record): IAirtableFilterGroup | null => {
+ for (const value of Object.values(typeOptions)) {
+ if (
+ value &&
+ typeof value === 'object' &&
+ Array.isArray((value as { filterSet?: unknown }).filterSet)
+ ) {
+ return value as IAirtableFilterGroup;
+ }
+ }
+ return null;
+};
+
+export class AirtableShareClient {
+ private lastRequestAt = 0;
+ private session?: IAirtableShareSession;
+
+ /** appId carried by a canonical share URL, for a cheap client-side pre-check. */
+ static parseBaseIdFromLink(shareLink: string): string | undefined {
+ return shareLink.match(/app[A-Za-z0-9]+/)?.[0];
+ }
+
+ /**
+ * Resolves a public shared-base link into a session by scraping the share
+ * page. Throws AirtableShareError with a typed reason on every failure mode.
+ */
+ async resolveShare(shareLink: string): Promise {
+ const { html, cookie } = await this.fetchSharePage(this.buildShareUrl(shareLink));
+ const session = this.parseShareSession(html, cookie);
+ this.session = session;
+ return session;
+ }
+
+ private async fetchSharePage(shareUrl: string): Promise<{ html: string; cookie: string }> {
+ let response: Response;
+ try {
+ response = await fetch(shareUrl, {
+ // eslint-disable-next-line @typescript-eslint/naming-convention -- HTTP header name
+ headers: { 'User-Agent': browserUserAgent, Accept: 'text/html' },
+ redirect: 'manual',
+ });
+ } catch (e) {
+ throw new AirtableShareError(
+ `Failed to reach Airtable: ${e instanceof Error ? e.message : 'network error'}`,
+ 'resolve_failed'
+ );
+ }
+ if (response.status >= 300 && response.status < 400) {
+ const location = response.headers.get('location') ?? '';
+ if (location.startsWith('/login')) {
+ throw new AirtableShareError(
+ 'The shared base is not public; turn on the public shared-base link in Airtable.',
+ 'requires_auth'
+ );
+ }
+ throw new AirtableShareError('The share link is not publicly accessible.', 'not_public');
+ }
+ if (!response.ok) {
+ throw new AirtableShareError(
+ `The share link is not accessible (status ${response.status}).`,
+ 'not_public'
+ );
+ }
+ const cookie = (response.headers.getSetCookie?.() ?? [])
+ .map((entry) => entry.split(';')[0])
+ .join('; ');
+ return { html: await response.text(), cookie };
+ }
+
+ private parseShareSession(html: string, cookie: string): IAirtableShareSession {
+ const requestId = html.match(/requestId: "(.*?)",/)?.[1];
+ const initRaw = html.match(/window\.initData = (.*?);\n/)?.[1];
+ if (!requestId || !initRaw) {
+ throw new AirtableShareError(
+ 'The link does not look like an Airtable shared base.',
+ 'not_a_base'
+ );
+ }
+ let initData: {
+ sharedApplicationId?: string;
+ accessPolicy?: unknown;
+ pageLoadId?: string;
+ codeVersion?: string;
+ };
+ try {
+ initData = JSON.parse(initRaw);
+ } catch {
+ throw new AirtableShareError('Failed to read the shared base metadata.', 'resolve_failed');
+ }
+ const appId = initData.sharedApplicationId;
+ if (!appId || typeof initData.accessPolicy !== 'string') {
+ throw new AirtableShareError(
+ 'The link is a shared view, not a shared base. Share the whole base instead.',
+ 'not_a_base'
+ );
+ }
+ return {
+ appId,
+ accessPolicy: initData.accessPolicy,
+ requestId,
+ pageLoadId: initData.pageLoadId ?? '',
+ codeVersion: initData.codeVersion ?? '',
+ cookie,
+ };
+ }
+
+ /** Asserts the resolved share points at the base the import is creating. */
+ assertBaseMatch(expectedBaseId: string): void {
+ if (!this.session) {
+ throw new AirtableShareError(sessionNotResolvedMessage, 'resolve_failed');
+ }
+ if (this.session.appId !== expectedBaseId) {
+ throw new AirtableShareError(
+ `The share link points at a different base (${this.session.appId}) than the one being imported (${expectedBaseId}).`,
+ 'base_mismatch'
+ );
+ }
+ }
+
+ /**
+ * Reads one view's configuration. The response also carries the view's full
+ * row ordering; those branches are dropped at the token level so memory stays
+ * flat regardless of how many records the view holds.
+ */
+ async fetchViewConfig(viewId: string): Promise {
+ const session = this.session;
+ if (!session) {
+ throw new AirtableShareError(sessionNotResolvedMessage, 'resolve_failed');
+ }
+ await this.throttle();
+
+ const params = new URLSearchParams({
+ stringifiedObjectParams: JSON.stringify({
+ mayOnlyIncludeRowAndCellDataForIncludedViews: true,
+ mayExcludeCellDataForLargeViews: true,
+ allowMsgpackOfResult: false,
+ }),
+ requestId: session.requestId,
+ accessPolicy: session.accessPolicy,
+ });
+ const response = await fetch(
+ `${shareApiBaseUrl}/view/${encodeURIComponent(viewId)}/readData?${params.toString()}`,
+ { headers: this.apiHeaders(session) }
+ );
+ if (!response.ok || !response.body) {
+ throw new AirtableShareError(
+ `Failed to read view ${viewId} (status ${response.status}).`,
+ 'resolve_failed'
+ );
+ }
+
+ const root = await this.assembleJson(
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ Readable.fromWeb(response.body as any),
+ /(?:^|\.)(rowOrder|signedUserContentUrls)(?:\.|$)/
+ );
+ const data = (root?.data ?? {}) as {
+ filters?: IAirtableFilterGroup;
+ lastSortsApplied?: { sortSet?: IAirtableSort[] };
+ groupLevels?: IAirtableGroupLevel[];
+ metadata?: Record;
+ };
+ return {
+ filters: data.filters ?? null,
+ sorts: data.lastSortsApplied?.sortSet ?? null,
+ groupLevels: data.groupLevels ?? null,
+ metadata: data.metadata,
+ };
+ }
+
+ /**
+ * Reads the whole base model — which the official API never exposes enough of
+ * to recreate rollups — and returns each rollup's link/foreign field,
+ * aggregation, and optional filter. The model also embeds every record
+ * (`tableDatas`); that branch is dropped at the token level so memory stays
+ * flat regardless of base size.
+ */
+ // eslint-disable-next-line sonarjs/cognitive-complexity -- flat schema walk
+ async fetchApplicationModel(): Promise> {
+ const session = this.session;
+ if (!session) {
+ throw new AirtableShareError(sessionNotResolvedMessage, 'resolve_failed');
+ }
+ await this.throttle();
+
+ const params = new URLSearchParams({
+ stringifiedObjectParams: JSON.stringify({}),
+ requestId: session.requestId,
+ accessPolicy: session.accessPolicy,
+ });
+ const response = await fetch(
+ `${shareApiBaseUrl}/application/${encodeURIComponent(session.appId)}/read?${params.toString()}`,
+ { headers: this.apiHeaders(session) }
+ );
+ if (!response.ok || !response.body) {
+ throw new AirtableShareError(
+ `Failed to read the base model (status ${response.status}).`,
+ 'resolve_failed'
+ );
+ }
+
+ const root = await this.assembleJson(
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ Readable.fromWeb(response.body as any),
+ /(?:^|\.)(tableDatas)(?:\.|$)/
+ );
+ const tableSchemas = ((root?.data as { tableSchemas?: unknown })?.tableSchemas ?? []) as Array<{
+ columns?: Array<{ id?: string; type?: string; typeOptions?: Record }>;
+ }>;
+
+ const rollups = new Map();
+ for (const table of tableSchemas) {
+ for (const column of table.columns ?? []) {
+ if (column.type !== 'rollup' || !column.id || !column.typeOptions) continue;
+ const { relationColumnId, foreignTableRollupColumnId, formulaTextParsed } =
+ column.typeOptions;
+ if (
+ typeof relationColumnId !== 'string' ||
+ typeof foreignTableRollupColumnId !== 'string' ||
+ typeof formulaTextParsed !== 'string'
+ ) {
+ continue;
+ }
+ rollups.set(column.id, {
+ relationColumnId,
+ foreignTableRollupColumnId,
+ aggregation: formulaTextParsed,
+ filter: findAirtableFilter(column.typeOptions),
+ });
+ }
+ }
+ return rollups;
+ }
+
+ /** Assembles the JSON value while dropping the matched (large) payload branches. */
+ private async assembleJson(
+ stream: Readable,
+ ignoreBranches: RegExp
+ ): Promise> {
+ const tokens = stream.pipe(parser()).pipe(ignore({ filter: ignoreBranches }));
+ const assembler = Assembler.connectTo(tokens);
+ await finished(tokens);
+ return (assembler.current ?? {}) as Record;
+ }
+
+ /**
+ * Builds the canonical `airtable.com/{appId}/{shrId}` URL. The share id alone
+ * (without the app prefix) 404s; the prefixed form is what Airtable's own
+ * "Copy link" produces and what other importers request.
+ */
+ private buildShareUrl(shareLink: string): string {
+ const shareId = shareLink.match(/shr[A-Za-z0-9]+/)?.[0];
+ if (!shareId) {
+ throw new AirtableShareError(
+ 'Enter a valid Airtable shared-base link (https://airtable.com/.../shr...).',
+ 'not_a_base'
+ );
+ }
+ const appId = shareLink.match(/app[A-Za-z0-9]+/)?.[0];
+ return appId ? `${shareBaseUrl}/${appId}/${shareId}` : `${shareBaseUrl}/${shareId}`;
+ }
+
+ private apiHeaders(session: IAirtableShareSession): Record {
+ /* eslint-disable @typescript-eslint/naming-convention -- HTTP header names */
+ return {
+ 'User-Agent': browserUserAgent,
+ Accept: 'application/json',
+ 'x-airtable-application-id': session.appId,
+ 'x-airtable-inter-service-client': 'webClient',
+ 'x-airtable-inter-service-client-code-version': session.codeVersion,
+ 'x-airtable-page-load-id': session.pageLoadId,
+ 'X-Requested-With': 'XMLHttpRequest',
+ 'x-time-zone': 'UTC',
+ 'x-user-locale': 'en',
+ cookie: session.cookie,
+ };
+ /* eslint-enable @typescript-eslint/naming-convention */
+ }
+
+ private async throttle(): Promise {
+ const wait = this.lastRequestAt + minRequestIntervalMs - Date.now();
+ if (wait > 0) {
+ await sleep(wait);
+ }
+ this.lastRequestAt = Date.now();
+ }
+}
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-token-resolver.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-token-resolver.ts
new file mode 100644
index 0000000000..cc74ec8793
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-token-resolver.ts
@@ -0,0 +1,16 @@
+/**
+ * Resolves the Airtable access token of a connected user integration on the
+ * server, so OAuth tokens never travel through the browser. The community
+ * edition has no integration storage; the enterprise backend provides an
+ * implementation through this injection token (optional dependency).
+ */
+export const AIRTABLE_IMPORT_TOKEN_RESOLVER = 'AIRTABLE_IMPORT_TOKEN_RESOLVER';
+
+export interface IAirtableImportTokenResolver {
+ /**
+ * Returns a currently valid access token for the given integration of the
+ * requesting user (refreshing it server-side when expired). Must reject
+ * when the integration does not exist or belongs to another user.
+ */
+ resolveAccessToken(integrationId: string): Promise;
+}
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-view-config-mapper.spec.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-view-config-mapper.spec.ts
new file mode 100644
index 0000000000..e0c9e58276
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-view-config-mapper.spec.ts
@@ -0,0 +1,152 @@
+import { CellValueType, FieldType, RowHeightLevel, ViewType } from '@teable/core';
+import type { IImportAirtableIssue } from '@teable/openapi';
+import { describe, expect, it } from 'vitest';
+import type { IAirtableViewConfig } from './airtable-share.client';
+import {
+ mapAirtableViewConfig,
+ type IImportFieldMeta,
+ type IViewConfigMapperContext,
+} from './airtable-view-config-mapper';
+
+const field = (
+ fieldId: string,
+ type: FieldType,
+ cellValueType: CellValueType,
+ isMultipleCellValue = false
+): IImportFieldMeta => ({ fieldId, type, cellValueType, isMultipleCellValue });
+
+const fields: Record = {
+ colText: field('fldText', FieldType.SingleLineText, CellValueType.String),
+ colNum: field('fldNum', FieldType.Number, CellValueType.Number),
+ colSel: field('fldSel', FieldType.SingleSelect, CellValueType.String),
+ colDate: field('fldDate', FieldType.Date, CellValueType.DateTime),
+ colUser: field('fldUser', FieldType.User, CellValueType.String),
+ colLink: field('fldLink', FieldType.Link, CellValueType.String, true),
+ colAtt: field('fldAtt', FieldType.Attachment, CellValueType.String, true),
+};
+
+const ctx: IViewConfigMapperContext = {
+ resolveField: (columnId) => fields[columnId],
+ resolveSelectOptionName: (columnId, optionId) =>
+ columnId === 'colSel' ? { optHigh: 'High', optLow: 'Low' }[optionId] : undefined,
+};
+
+const emptyConfig = (over: Partial = {}): IAirtableViewConfig => ({
+ filters: null,
+ sorts: null,
+ groupLevels: null,
+ metadata: undefined,
+ ...over,
+});
+
+const run = (config: IAirtableViewConfig, teableViewType = ViewType.Grid) => {
+ const issues: IImportAirtableIssue[] = [];
+ const result = mapAirtableViewConfig({
+ teableViewType,
+ config,
+ ctx,
+ tableName: 'T',
+ viewName: 'V',
+ issues,
+ });
+ return { result, issues };
+};
+
+const filterOf = (...leaves: Array<{ columnId: string; operator: string; value: unknown }>) =>
+ emptyConfig({ filters: { conjunction: 'and', filterSet: leaves } });
+
+describe('mapAirtableViewConfig filters', () => {
+ it('maps text, number and select operators with mapped field ids and option names', () => {
+ const { result } = run(
+ filterOf(
+ { columnId: 'colText', operator: 'contains', value: '1' },
+ { columnId: 'colNum', operator: '>', value: 5 },
+ { columnId: 'colSel', operator: 'isAnyOf', value: ['optHigh', 'optLow'] }
+ )
+ );
+ expect(result.filter).toEqual({
+ conjunction: 'and',
+ filterSet: [
+ { fieldId: 'fldText', operator: 'contains', value: '1' },
+ { fieldId: 'fldNum', operator: 'isGreater', value: 5 },
+ { fieldId: 'fldSel', operator: 'isAnyOf', value: ['High', 'Low'] },
+ ],
+ });
+ });
+
+ it('keeps empty/not-empty with a null value and maps a date condition to a mode object', () => {
+ const { result } = run(
+ filterOf(
+ { columnId: 'colText', operator: 'isEmpty', value: null },
+ { columnId: 'colDate', operator: '=', value: { mode: 'today' } }
+ )
+ );
+ expect(result.filter?.filterSet).toEqual([
+ { fieldId: 'fldText', operator: 'isEmpty', value: null },
+ { fieldId: 'fldDate', operator: 'is', value: { mode: 'today', timeZone: 'UTC' } },
+ ]);
+ });
+
+ it('drops conditions that cannot be converted and reports them, never guessing', () => {
+ // attachment filename has no Teable equivalent; a link condition references
+ // specific records that cannot be remapped; an unknown field is skipped.
+ const { result, issues } = run(
+ filterOf(
+ { columnId: 'colAtt', operator: 'filename', value: 'a.png' },
+ { columnId: 'colLink', operator: 'isAnyOf', value: ['rec1'] },
+ { columnId: 'colLink', operator: 'isEmpty', value: null },
+ { columnId: 'colGone', operator: 'contains', value: 'x' }
+ )
+ );
+ // Only the link "is empty" condition survives.
+ expect(result.filter?.filterSet).toEqual([
+ { fieldId: 'fldLink', operator: 'isEmpty', value: null },
+ ]);
+ expect(issues.every((issue) => issue.code === 'viewConfigDegraded')).toBe(true);
+ expect(issues.length).toBeGreaterThanOrEqual(3);
+ });
+});
+
+describe('mapAirtableViewConfig sort, group and options', () => {
+ it('maps sorts and grid grouping by order', () => {
+ const { result } = run(
+ emptyConfig({
+ sorts: [
+ { columnId: 'colText', ascending: true },
+ { columnId: 'colNum', ascending: false },
+ ],
+ groupLevels: [{ columnId: 'colSel', order: 'descending' }],
+ })
+ );
+ expect(result.sort).toEqual({
+ sortObjs: [
+ { fieldId: 'fldText', order: 'asc' },
+ { fieldId: 'fldNum', order: 'desc' },
+ ],
+ manualSort: false,
+ });
+ expect(result.group).toEqual([{ fieldId: 'fldSel', order: 'desc' }]);
+ });
+
+ it('reads the kanban stack from the group level (a collaborator field, not a guess)', () => {
+ const { result } = run(
+ emptyConfig({
+ groupLevels: [{ columnId: 'colUser', order: 'ascending' }],
+ metadata: { kanban: { coverColumnId: 'colAtt' } },
+ }),
+ ViewType.Kanban
+ );
+ // For kanban the group level is the stack field; no row grouping is emitted.
+ expect(result.group).toBeUndefined();
+ expect(result.options).toEqual({ stackFieldId: 'fldUser', coverFieldId: 'fldAtt' });
+ });
+
+ it('maps known grid row heights and degrades unknown ones', () => {
+ expect(run(emptyConfig({ metadata: { grid: { rowHeight: 'tall' } } })).result.options).toEqual({
+ rowHeight: RowHeightLevel.Tall,
+ });
+ const unknown = run(emptyConfig({ metadata: { grid: { rowHeight: 'gigantic' } } }));
+ expect(unknown.result.options).toBeUndefined();
+ expect(unknown.issues[0]?.code).toBe('viewConfigDegraded');
+ });
+});
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable-view-config-mapper.ts b/apps/nestjs-backend/src/features/airtable-import/airtable-view-config-mapper.ts
new file mode 100644
index 0000000000..50e7859e06
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable-view-config-mapper.ts
@@ -0,0 +1,396 @@
+import type {
+ IDateTimeFieldOperator,
+ IFilter,
+ IFilterItem,
+ IFilterSet,
+ IGroup,
+ IOperator,
+ ISort,
+ ISortItem,
+} from '@teable/core';
+import {
+ CellValueType,
+ FieldType,
+ getValidFilterOperators,
+ getValidFilterSubOperators,
+ RowHeightLevel,
+ SortFunc,
+ ViewType,
+} from '@teable/core';
+import type { IImportAirtableIssue } from '@teable/openapi';
+import type {
+ IAirtableFilterGroup,
+ IAirtableFilterLeaf,
+ IAirtableViewConfig,
+} from './airtable-share.client';
+
+/** The Teable field a mapped view references, with what filtering needs. */
+export interface IImportFieldMeta {
+ fieldId: string;
+ type: FieldType;
+ cellValueType: CellValueType;
+ isMultipleCellValue: boolean;
+}
+
+export interface IViewConfigMapperContext {
+ /** airtable columnId (a field id) -> imported Teable field, or undefined. */
+ resolveField: (columnId: string) => IImportFieldMeta | undefined;
+ /** airtable select option id -> its name (Teable filters select by name). */
+ resolveSelectOptionName: (columnId: string, optionId: string) => string | undefined;
+}
+
+export interface IMappedViewConfig {
+ filter?: IFilter;
+ sort?: ISort;
+ group?: IGroup;
+ options?: Record;
+}
+
+/** Airtable date-filter mode -> Teable date sub-operator (mode). */
+const dateModeMap: Record = {
+ today: 'today',
+ tomorrow: 'tomorrow',
+ yesterday: 'yesterday',
+ exactDate: 'exactDate',
+ daysAgo: 'daysAgo',
+ daysFromNow: 'daysFromNow',
+ oneWeekAgo: 'oneWeekAgo',
+ oneWeekFromNow: 'oneWeekFromNow',
+ oneMonthAgo: 'oneMonthAgo',
+ oneMonthFromNow: 'oneMonthFromNow',
+ pastWeek: 'pastWeek',
+ pastMonth: 'pastMonth',
+ pastYear: 'pastYear',
+ nextWeek: 'nextWeek',
+ nextMonth: 'nextMonth',
+ nextYear: 'nextYear',
+ pastNumberOfDays: 'pastNumberOfDays',
+ nextNumberOfDays: 'nextNumberOfDays',
+ thisCalendarWeek: 'currentWeek',
+ thisCalendarMonth: 'currentMonth',
+ thisCalendarYear: 'currentYear',
+};
+
+/** Airtable rowHeight -> Teable RowHeightLevel; unknown -> undefined (degrade). */
+const rowHeightMap: Record = {
+ short: RowHeightLevel.Short,
+ medium: RowHeightLevel.Medium,
+ tall: RowHeightLevel.Tall,
+ extraTall: RowHeightLevel.ExtraTall,
+ // legacy alias seen in other importers
+ xlarge: RowHeightLevel.ExtraTall,
+};
+
+const isSelectField = (meta: IImportFieldMeta) =>
+ meta.type === FieldType.SingleSelect || meta.type === FieldType.MultipleSelect;
+
+const isMultiValued = (meta: IImportFieldMeta) =>
+ meta.isMultipleCellValue === true || meta.type === FieldType.MultipleSelect;
+
+/**
+ * Fields whose Airtable filter value references specific records/collaborators
+ * (link, user, ...). Those ids cannot be remapped reliably at view-config time,
+ * so value-based conditions on them are dropped (empty/not-empty still apply).
+ */
+const isRecordReferenceField = (meta: IImportFieldMeta) =>
+ meta.type === FieldType.Link ||
+ meta.type === FieldType.User ||
+ meta.type === FieldType.CreatedBy ||
+ meta.type === FieldType.LastModifiedBy;
+
+// Operator dispatch: branchy by nature (field type x Airtable operator).
+// eslint-disable-next-line sonarjs/cognitive-complexity
+const mapOperator = (airtableOperator: string, meta: IImportFieldMeta): IOperator | undefined => {
+ if (airtableOperator === 'isEmpty') return 'isEmpty';
+ if (airtableOperator === 'isNotEmpty') return 'isNotEmpty';
+
+ const isDate = meta.cellValueType === CellValueType.DateTime;
+ const multi = isMultiValued(meta);
+
+ switch (airtableOperator) {
+ case '=':
+ if (isDate) return 'is';
+ return multi ? 'isExactly' : 'is';
+ case '!=':
+ if (isDate) return 'isNot';
+ return multi ? 'isNotExactly' : 'isNot';
+ case '<':
+ return isDate ? 'isBefore' : 'isLess';
+ case '<=':
+ return isDate ? 'isOnOrBefore' : 'isLessEqual';
+ case '>':
+ return isDate ? 'isAfter' : 'isGreater';
+ case '>=':
+ return isDate ? 'isOnOrAfter' : 'isGreaterEqual';
+ case 'contains':
+ return 'contains';
+ case 'doesNotContain':
+ return 'doesNotContain';
+ case 'isWithin':
+ return 'isWithIn';
+ case 'isAnyOf':
+ case '|':
+ return multi ? 'hasAnyOf' : 'isAnyOf';
+ case 'isNoneOf':
+ return multi ? 'hasNoneOf' : 'isNoneOf';
+ case '&':
+ return multi ? 'hasAllOf' : 'isAnyOf';
+ default:
+ // filename, filetype, and any unknown operator
+ return undefined;
+ }
+};
+
+const needsNoValue = (operator: IOperator) => operator === 'isEmpty' || operator === 'isNotEmpty';
+
+/** Maps Airtable select option id(s) to the option name(s) Teable filters by. */
+const mapSelectFilterValue = (
+ leaf: IAirtableFilterLeaf,
+ operator: IOperator,
+ meta: IImportFieldMeta,
+ ctx: IViewConfigMapperContext
+): { value: IFilterItem['value'] } | undefined => {
+ const ids = Array.isArray(leaf.value) ? leaf.value : [leaf.value];
+ const names = ids
+ .map((id) =>
+ typeof id === 'string' ? ctx.resolveSelectOptionName(leaf.columnId, id) : undefined
+ )
+ .filter((name): name is string => name != null);
+ if (names.length === 0) return undefined;
+ const multi = isMultiValued(meta) || operator === 'isAnyOf' || operator === 'isNoneOf';
+ return { value: multi ? names : names[0] };
+};
+
+/** Maps a scalar (number/boolean/text) Airtable filter value to Teable's. */
+const mapScalarFilterValue = (
+ value: unknown,
+ cellValueType: CellValueType
+): { value: IFilterItem['value'] } | undefined => {
+ if (cellValueType === CellValueType.Boolean) {
+ return { value: value === true || value === 'true' };
+ }
+ if (cellValueType === CellValueType.Number) {
+ const num = typeof value === 'number' ? value : Number(value);
+ return Number.isFinite(num) ? { value: num } : undefined;
+ }
+ if (typeof value === 'string' || typeof value === 'number') {
+ return { value: String(value) };
+ }
+ return undefined;
+};
+
+/** Drops a clause (returns undefined) or yields the mapped Teable value. */
+const mapFilterValue = (
+ leaf: IAirtableFilterLeaf,
+ operator: IOperator,
+ meta: IImportFieldMeta,
+ ctx: IViewConfigMapperContext
+): { value: IFilterItem['value'] } | undefined => {
+ if (needsNoValue(operator)) return { value: null };
+ // Conditions that point at specific linked records/collaborators can't be
+ // remapped to the new ids here; keep the import honest by dropping them.
+ if (isRecordReferenceField(meta)) return undefined;
+ if (isSelectField(meta)) return mapSelectFilterValue(leaf, operator, meta, ctx);
+ if (meta.cellValueType === CellValueType.DateTime) {
+ return mapDateValue(leaf.value, operator, meta);
+ }
+ return mapScalarFilterValue(leaf.value, meta.cellValueType);
+};
+
+const mapDateValue = (
+ rawValue: unknown,
+ operator: IOperator,
+ meta: IImportFieldMeta
+): { value: IFilterItem['value'] } | undefined => {
+ if (!rawValue || typeof rawValue !== 'object') return undefined;
+ const source = rawValue as { mode?: string; numberOfDays?: number; exactDate?: string };
+ const teableMode = source.mode ? dateModeMap[source.mode] : undefined;
+ if (!teableMode) return undefined;
+
+ const validModes = getValidFilterSubOperators(meta.type, operator as IDateTimeFieldOperator);
+ if (!validModes || !validModes.includes(teableMode as never)) return undefined;
+
+ const value: Record = { mode: teableMode, timeZone: 'UTC' };
+ if (source.numberOfDays != null) value.numberOfDays = source.numberOfDays;
+ if (source.exactDate != null) value.exactDate = source.exactDate;
+ return { value: value as IFilterItem['value'] };
+};
+
+const mapFilterLeaf = (
+ leaf: IAirtableFilterLeaf,
+ ctx: IViewConfigMapperContext,
+ onDrop: (reason: string) => void
+): IFilterItem | undefined => {
+ const meta = ctx.resolveField(leaf.columnId);
+ if (!meta) {
+ onDrop('a filter field was not imported');
+ return undefined;
+ }
+ const operator = mapOperator(leaf.operator, meta);
+ if (!operator || !getValidFilterOperators(meta).includes(operator)) {
+ onDrop(`operator "${leaf.operator}" is not supported on this field`);
+ return undefined;
+ }
+ const mapped = mapFilterValue(leaf, operator, meta, ctx);
+ if (!mapped) {
+ onDrop(`a "${leaf.operator}" condition could not be converted`);
+ return undefined;
+ }
+ return { fieldId: meta.fieldId, operator, value: mapped.value };
+};
+
+const isFilterGroup = (
+ node: IAirtableFilterLeaf | IAirtableFilterGroup
+): node is IAirtableFilterGroup =>
+ (node as IAirtableFilterGroup).filterSet !== undefined &&
+ (node as IAirtableFilterGroup).conjunction !== undefined;
+
+const mapFilterNode = (
+ node: IAirtableFilterGroup,
+ ctx: IViewConfigMapperContext,
+ onDrop: (reason: string) => void
+): IFilterSet | undefined => {
+ const filterSet: Array = [];
+ for (const child of node.filterSet ?? []) {
+ if (isFilterGroup(child)) {
+ const nested = mapFilterNode(child, ctx, onDrop);
+ if (nested && nested.filterSet.length > 0) filterSet.push(nested);
+ } else {
+ const leaf = mapFilterLeaf(child, ctx, onDrop);
+ if (leaf) filterSet.push(leaf);
+ }
+ }
+ if (filterSet.length === 0) return undefined;
+ return { conjunction: node.conjunction === 'or' ? 'or' : 'and', filterSet };
+};
+
+/**
+ * Maps an Airtable filter group to a Teable filter. Shared by view-config import
+ * and rollup record-selection ("only include linked records that meet conditions").
+ */
+export const mapAirtableFilter = (
+ filter: IAirtableFilterGroup,
+ ctx: IViewConfigMapperContext,
+ onDrop: (reason: string) => void = () => undefined
+): IFilter | undefined => mapFilterNode(filter, ctx, onDrop);
+
+const mapSorts = (
+ config: IAirtableViewConfig,
+ ctx: IViewConfigMapperContext
+): ISort | undefined => {
+ const sortObjs: ISortItem[] = [];
+ for (const sort of config.sorts ?? []) {
+ const meta = ctx.resolveField(sort.columnId);
+ if (!meta) continue;
+ sortObjs.push({ fieldId: meta.fieldId, order: sort.ascending ? SortFunc.Asc : SortFunc.Desc });
+ }
+ return sortObjs.length > 0 ? { sortObjs, manualSort: false } : undefined;
+};
+
+const mapGroups = (
+ config: IAirtableViewConfig,
+ ctx: IViewConfigMapperContext
+): IGroup | undefined => {
+ const group = (config.groupLevels ?? [])
+ .map((level) => {
+ const meta = ctx.resolveField(level.columnId);
+ if (!meta) return undefined;
+ return {
+ fieldId: meta.fieldId,
+ order: level.order === 'descending' ? SortFunc.Desc : SortFunc.Asc,
+ };
+ })
+ .filter((item): item is { fieldId: string; order: SortFunc } => item != null);
+ return group.length > 0 ? group : undefined;
+};
+
+const metadataFieldId = (
+ metadata: Record | undefined,
+ path: string[],
+ ctx: IViewConfigMapperContext
+): string | undefined => {
+ let node: unknown = metadata;
+ for (const key of path) {
+ if (!node || typeof node !== 'object') return undefined;
+ node = (node as Record)[key];
+ }
+ return typeof node === 'string' ? ctx.resolveField(node)?.fieldId : undefined;
+};
+
+const kanbanOptions = (config: IAirtableViewConfig, ctx: IViewConfigMapperContext) => {
+ const options: Record = {};
+ // Airtable models the kanban "stack by" as the single group level.
+ const stackColumnId = config.groupLevels?.[0]?.columnId;
+ const stackFieldId = stackColumnId ? ctx.resolveField(stackColumnId)?.fieldId : undefined;
+ if (stackFieldId) options.stackFieldId = stackFieldId;
+ const coverFieldId = metadataFieldId(config.metadata, ['kanban', 'coverColumnId'], ctx);
+ if (coverFieldId) options.coverFieldId = coverFieldId;
+ return options;
+};
+
+const calendarOptions = (config: IAirtableViewConfig, ctx: IViewConfigMapperContext) => {
+ const ranges = (
+ config.metadata?.calendar as { dateColumnRanges?: Array<{ startColumnId?: string }> }
+ )?.dateColumnRanges;
+ const startColumnId = ranges?.[0]?.startColumnId;
+ const startDateFieldId = startColumnId ? ctx.resolveField(startColumnId)?.fieldId : undefined;
+ return startDateFieldId ? { startDateFieldId } : {};
+};
+
+const gridOptions = (config: IAirtableViewConfig, onDrop: (reason: string) => void) => {
+ const rawRowHeight = (config.metadata?.grid as { rowHeight?: string })?.rowHeight;
+ if (!rawRowHeight) return {};
+ const rowHeight = rowHeightMap[rawRowHeight];
+ if (rowHeight) return { rowHeight };
+ onDrop(`row height "${rawRowHeight}" is not supported`);
+ return {};
+};
+
+const mapOptions = (
+ teableViewType: ViewType,
+ config: IAirtableViewConfig,
+ ctx: IViewConfigMapperContext,
+ onDrop: (reason: string) => void
+): Record | undefined => {
+ const builders: Partial Record>> = {
+ [ViewType.Kanban]: () => kanbanOptions(config, ctx),
+ [ViewType.Gallery]: () => {
+ const coverFieldId = metadataFieldId(config.metadata, ['gallery', 'coverColumnId'], ctx);
+ return coverFieldId ? { coverFieldId } : {};
+ },
+ [ViewType.Calendar]: () => calendarOptions(config, ctx),
+ [ViewType.Grid]: () => gridOptions(config, onDrop),
+ };
+ const options = builders[teableViewType]?.() ?? {};
+ return Object.keys(options).length > 0 ? options : undefined;
+};
+
+/**
+ * Maps one Airtable view's configuration to Teable filter/sort/group/options.
+ * Anything that cannot be converted faithfully is dropped and reported, never
+ * guessed — the core import is unaffected.
+ */
+export const mapAirtableViewConfig = (params: {
+ teableViewType: ViewType;
+ config: IAirtableViewConfig;
+ ctx: IViewConfigMapperContext;
+ tableName: string;
+ viewName: string;
+ issues: IImportAirtableIssue[];
+}): IMappedViewConfig => {
+ const { teableViewType, config, ctx, tableName, viewName, issues } = params;
+ const droppedReasons = new Set();
+ const onDrop = (reason: string) => droppedReasons.add(reason);
+
+ const filter = config.filters ? mapFilterNode(config.filters, ctx, onDrop) : undefined;
+ const sort = mapSorts(config, ctx);
+ // For kanban the single group level is the stack field, not a row grouping.
+ const group = teableViewType === ViewType.Kanban ? undefined : mapGroups(config, ctx);
+ const options = mapOptions(teableViewType, config, ctx, onDrop);
+
+ for (const reason of droppedReasons) {
+ issues.push({ code: 'viewConfigDegraded', tableName, viewName, reason });
+ }
+
+ return { filter: filter ?? undefined, sort, group, options };
+};
diff --git a/apps/nestjs-backend/src/features/airtable-import/airtable.types.ts b/apps/nestjs-backend/src/features/airtable-import/airtable.types.ts
new file mode 100644
index 0000000000..5e598e39cb
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/airtable.types.ts
@@ -0,0 +1,124 @@
+/**
+ * Typings for the subset of the official Airtable Web API used by the importer.
+ * https://airtable.com/developers/web/api
+ */
+
+export interface IAirtableBaseItem {
+ id: string;
+ name: string;
+ permissionLevel: 'none' | 'read' | 'comment' | 'edit' | 'create';
+}
+
+export interface IAirtableListBasesResponse {
+ bases: IAirtableBaseItem[];
+ offset?: string;
+}
+
+export interface IAirtableSelectChoice {
+ id: string;
+ name: string;
+ color?: string;
+}
+
+export interface IAirtableFieldResult {
+ type: string;
+ options?: Record;
+}
+
+/**
+ * Per-type options. The official API documents one shape per field type; we
+ * keep a single permissive interface because the mapper only reads a few keys
+ * per type and unknown field types must not break the import.
+ */
+export interface IAirtableFieldOptions {
+ // number / percent / currency
+ precision?: number;
+ symbol?: string;
+ // rating / checkbox
+ max?: number;
+ icon?: string;
+ color?: string;
+ // duration
+ durationFormat?: string;
+ // date / dateTime
+ dateFormat?: { name?: string; format?: string };
+ timeFormat?: { name?: string; format?: string };
+ timeZone?: string;
+ // select-like
+ choices?: IAirtableSelectChoice[];
+ // multipleRecordLinks
+ linkedTableId?: string;
+ prefersSingleRecordLink?: boolean;
+ inverseLinkFieldId?: string;
+ isReversed?: boolean;
+ /** "Limit record selection to a view" — the linked table's view id, when set. */
+ viewIdForRecordSelection?: string;
+ // lookup / rollup / count
+ recordLinkFieldId?: string | null;
+ fieldIdInLinkedTable?: string | null;
+ isValid?: boolean;
+ result?: IAirtableFieldResult | null;
+ // formula
+ formula?: string;
+ referencedFieldIds?: string[] | null;
+ // aiText: array of text chunks and field reference objects
+ prompt?: unknown[];
+}
+
+export interface IAirtableField {
+ id: string;
+ name: string;
+ type: string;
+ description?: string;
+ options?: IAirtableFieldOptions;
+}
+
+export interface IAirtableView {
+ id: string;
+ name: string;
+ type: string;
+}
+
+export interface IAirtableTable {
+ id: string;
+ name: string;
+ description?: string;
+ primaryFieldId: string;
+ fields: IAirtableField[];
+ views: IAirtableView[];
+}
+
+export interface IAirtableBaseSchemaResponse {
+ tables: IAirtableTable[];
+}
+
+export interface IAirtableAttachment {
+ id: string;
+ url: string;
+ filename: string;
+ size?: number;
+ type?: string;
+}
+
+export interface IAirtableCollaborator {
+ id: string;
+ email?: string;
+ name?: string;
+}
+
+export interface IAirtableRecord {
+ id: string;
+ createdTime: string;
+ fields: Record;
+}
+
+export interface IAirtableListRecordsResponse {
+ records: IAirtableRecord[];
+ offset?: string;
+}
+
+export interface IAirtableApiError {
+ status: number;
+ type?: string;
+ message?: string;
+}
diff --git a/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-build-test-base.mjs b/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-build-test-base.mjs
new file mode 100644
index 0000000000..f8032bb4b8
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-build-test-base.mjs
@@ -0,0 +1,363 @@
+#!/usr/bin/env node
+/**
+ * Builds a comprehensive "kitchen sink" test base in Airtable via the official
+ * meta API, to exercise the Airtable -> Teable importer. Reproducible: delete
+ * the base and re-run anytime.
+ *
+ * Goal: hit as many field types AND option combinations as the API allows, so
+ * every mapping branch the importer reads is covered — number precision,
+ * currency symbol, rating icon/color/max, date format/time format/time zone,
+ * duration format, select colors + special names, checkbox icon/color, and
+ * (especially) the many LINK shapes: multi, single, self (multi + single),
+ * several links between the same two tables, and a one-to-many whose MULTI
+ * side is traversed first (the case that must still resolve to ManyOne).
+ *
+ * The API CANNOT create: computed fields (formula / rollup / lookup / count),
+ * button, autoNumber, createdTime/By, externalSyncSource, or custom views.
+ * Those are reported at the end for manual add.
+ *
+ * Usage:
+ * AIRTABLE_PAT=patXXX WORKSPACE=wspXXX node src/features/airtable-import/test-scripts/airtable-build-test-base.mjs
+ * (or BASE_ID=appXXX to add the tables to an existing base instead of creating one)
+ *
+ * PAT scopes: schema.bases:write, data.records:write, schema.bases:read, data.records:read.
+ */
+const PAT = process.env.AIRTABLE_PAT;
+const WORKSPACE = process.env.WORKSPACE;
+const BASE_NAME = 'Import Test — Kitchen Sink';
+const API = 'https://api.airtable.com/v0';
+if (!PAT || (!WORKSPACE && !process.env.BASE_ID)) {
+ console.error('Set AIRTABLE_PAT and WORKSPACE (wsp…), or BASE_ID (app…) to reuse a base.');
+ process.exit(1);
+}
+
+const req = async (method, path, body) => {
+ const res = await fetch(`${API}${path}`, {
+ method,
+ headers: { Authorization: `Bearer ${PAT}`, 'Content-Type': 'application/json' },
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ const text = await res.text();
+ const json = text ? JSON.parse(text) : null;
+ if (!res.ok) {
+ const err = new Error(`${method} ${path} -> ${res.status} ${text.slice(0, 300)}`);
+ err.status = res.status;
+ throw err;
+ }
+ return json;
+};
+
+const failedFields = [];
+const fieldIds = {}; // name -> id (only successfully created)
+const addField = async (baseId, tableId, field) => {
+ try {
+ const created = await req('POST', `/meta/bases/${baseId}/tables/${tableId}/fields`, field);
+ fieldIds[field.name] = created.id;
+ return created.id;
+ } catch (e) {
+ const reason = e.message.split('->')[1]?.trim() ?? e.message;
+ console.warn(` ⚠ "${field.name}" (${field.type}) skipped: ${reason}`);
+ failedFields.push(`${field.name} (${field.type}): ${reason}`);
+ return null;
+ }
+};
+
+const main = async () => {
+ // 1) Tables — three so links can be cross-table, self, and from a 3rd table.
+ let baseId;
+ let A; // All Fields
+ let B; // Linked
+ let C; // Self
+ if (process.env.BASE_ID) {
+ baseId = process.env.BASE_ID;
+ const mk = async (name, primary) =>
+ (
+ await req('POST', `/meta/bases/${baseId}/tables`, {
+ name,
+ fields: [{ name: primary, type: 'singleLineText' }],
+ })
+ ).id;
+ A = await mk('All Fields', 'Name');
+ B = await mk('Linked', 'Name');
+ C = await mk('Self', 'Title');
+ console.log(`using existing base ${baseId} (added All Fields/Linked/Self)`);
+ } else {
+ const base = await req('POST', '/meta/bases', {
+ workspaceId: WORKSPACE,
+ name: BASE_NAME,
+ tables: [
+ { name: 'All Fields', fields: [{ name: 'Name', type: 'singleLineText' }] },
+ { name: 'Linked', fields: [{ name: 'Name', type: 'singleLineText' }] },
+ { name: 'Self', fields: [{ name: 'Title', type: 'singleLineText' }] },
+ ],
+ });
+ baseId = base.id;
+ const tid = (name) => base.tables.find((t) => t.name === name).id;
+ A = tid('All Fields');
+ B = tid('Linked');
+ C = tid('Self');
+ console.log(`base ${baseId} created (All Fields=${A}, Linked=${B}, Self=${C})`);
+ }
+
+ // 2) Linked table — targets for lookups / rollups / counts (added manually).
+ await addField(baseId, B, { name: 'Amount', type: 'number', options: { precision: 2 } });
+ await addField(baseId, B, {
+ name: 'Category',
+ type: 'singleSelect',
+ options: { choices: [{ name: 'X' }, { name: 'Y' }, { name: 'Z' }] },
+ });
+ await addField(baseId, B, {
+ name: 'When',
+ type: 'date',
+ options: { dateFormat: { name: 'iso' } },
+ });
+ await addField(baseId, B, { name: 'Files', type: 'multipleAttachments' });
+
+ // 3) All Fields — one column per type, spread across option combinations.
+ const basicFields = [
+ // text family (email/url/phone get showAs on the Teable side)
+ { name: 'Long text', type: 'multilineText' },
+ { name: 'Rich text', type: 'richText' },
+ { name: 'Email', type: 'email' },
+ { name: 'URL', type: 'url' },
+ { name: 'Phone', type: 'phoneNumber' },
+ { name: 'Barcode', type: 'barcode' },
+ // number precision / percent / currency symbols
+ { name: 'Int', type: 'number', options: { precision: 0 } },
+ { name: 'Decimal 1', type: 'number', options: { precision: 1 } },
+ { name: 'Decimal 4', type: 'number', options: { precision: 4 } },
+ { name: 'Percent 0', type: 'percent', options: { precision: 0 } },
+ { name: 'Percent 2', type: 'percent', options: { precision: 2 } },
+ { name: 'Currency USD', type: 'currency', options: { precision: 2, symbol: '$' } },
+ { name: 'Currency EUR', type: 'currency', options: { precision: 0, symbol: '€' } },
+ { name: 'Currency CNY', type: 'currency', options: { precision: 2, symbol: '¥' } },
+ // duration formats
+ { name: 'Dur h:mm', type: 'duration', options: { durationFormat: 'h:mm' } },
+ { name: 'Dur h:mm:ss', type: 'duration', options: { durationFormat: 'h:mm:ss' } },
+ { name: 'Dur millis', type: 'duration', options: { durationFormat: 'h:mm:ss.SSS' } },
+ // rating icon / color / max
+ {
+ name: 'Rate star 5',
+ type: 'rating',
+ options: { max: 5, icon: 'star', color: 'yellowBright' },
+ },
+ {
+ name: 'Rate heart 10',
+ type: 'rating',
+ options: { max: 10, icon: 'heart', color: 'redBright' },
+ },
+ { name: 'Rate flag 3', type: 'rating', options: { max: 3, icon: 'flag', color: 'blueBright' } },
+ // date format variants
+ { name: 'Date iso', type: 'date', options: { dateFormat: { name: 'iso' } } },
+ { name: 'Date us', type: 'date', options: { dateFormat: { name: 'us' } } },
+ { name: 'Date euro', type: 'date', options: { dateFormat: { name: 'european' } } },
+ { name: 'Date friendly', type: 'date', options: { dateFormat: { name: 'friendly' } } },
+ // dateTime: format × timeFormat × timeZone
+ {
+ name: 'DT utc 24',
+ type: 'dateTime',
+ options: { dateFormat: { name: 'iso' }, timeFormat: { name: '24hour' }, timeZone: 'utc' },
+ },
+ {
+ name: 'DT client 12',
+ type: 'dateTime',
+ options: { dateFormat: { name: 'us' }, timeFormat: { name: '12hour' }, timeZone: 'client' },
+ },
+ {
+ name: 'DT shanghai',
+ type: 'dateTime',
+ options: {
+ dateFormat: { name: 'iso' },
+ timeFormat: { name: '24hour' },
+ timeZone: 'Asia/Shanghai',
+ },
+ },
+ // select: many colors + special option names (emoji, comma)
+ {
+ name: 'Single select',
+ type: 'singleSelect',
+ options: {
+ choices: [
+ { name: 'Alpha', color: 'blueBright' },
+ { name: 'Beta', color: 'greenBright' },
+ { name: 'Gamma', color: 'redBright' },
+ { name: 'Delta 🚀', color: 'purpleBright' },
+ { name: 'Has, comma', color: 'orangeBright' },
+ ],
+ },
+ },
+ {
+ name: 'Multi select',
+ type: 'multipleSelects',
+ options: {
+ choices: [
+ { name: 'One', color: 'cyanBright' },
+ { name: 'Two', color: 'tealBright' },
+ { name: 'Three', color: 'pinkBright' },
+ { name: 'Four', color: 'grayBright' },
+ ],
+ },
+ },
+ // checkbox icon / color
+ { name: 'Check green', type: 'checkbox', options: { icon: 'check', color: 'greenBright' } },
+ { name: 'Check heart', type: 'checkbox', options: { icon: 'heart', color: 'redBright' } },
+ // collaborators + attachments
+ { name: 'Single collaborator', type: 'singleCollaborator' },
+ { name: 'Multi collaborator', type: 'multipleCollaborators' },
+ { name: 'Attachments', type: 'multipleAttachments' },
+ // system / auto — API rejects these; reported for manual add
+ { name: 'Created time', type: 'createdTime' },
+ { name: 'Auto number', type: 'autoNumber' },
+ ];
+ for (const f of basicFields) await addField(baseId, A, f);
+
+ // 4) LINKS — the many shapes (this is where the cardinality bugs live).
+ // Each two-way link auto-creates a symmetric field on the other table.
+ // NOTE: the API cannot set prefersSingleRecordLink at create time, so every
+ // link below is created as MULTI. Toggle the single-link ones in the UI
+ // afterwards (see the manual list) to get the single + one-to-many shapes.
+ const links = [
+ // A -> B, two links between the SAME two tables (symmetric-collision test)
+ { t: A, name: 'Link to B', opt: { linkedTableId: B } },
+ { t: A, name: 'Link to B 2', opt: { linkedTableId: B } },
+ // self link on A
+ { t: A, name: 'Self link', opt: { linkedTableId: A } },
+ // a link created from B -> A (gives A an auto symmetric). Toggle THIS to
+ // single-link in the UI: A (table[0]) is then the multi side seen first —
+ // the one-to-many that must still resolve to ManyOne.
+ { t: B, name: 'Link to A', opt: { linkedTableId: A } },
+ // link from a third table
+ { t: C, name: 'C to A', opt: { linkedTableId: A } },
+ ];
+ for (const l of links)
+ await addField(baseId, l.t, { name: l.name, type: 'multipleRecordLinks', options: l.opt });
+
+ // 5) Records. B first (for ids), then A referencing them, then back/self links.
+ const createRecords = async (tableId, records) => {
+ const filtered = records.map((r) => ({
+ fields: Object.fromEntries(
+ Object.entries(r.fields).filter(
+ ([k]) => k === 'Name' || k === 'Title' || fieldIds[k] != null
+ )
+ ),
+ }));
+ const out = [];
+ for (let i = 0; i < filtered.length; i += 10) {
+ const res = await req('POST', `/${baseId}/${tableId}`, {
+ records: filtered.slice(i, i + 10),
+ typecast: true,
+ });
+ out.push(...res.records);
+ }
+ return out;
+ };
+ const patchRecords = async (tableId, records) => {
+ for (let i = 0; i < records.length; i += 10)
+ await req('PATCH', `/${baseId}/${tableId}`, {
+ records: records.slice(i, i + 10),
+ typecast: true,
+ });
+ };
+
+ const linkedRecs = await createRecords(
+ B,
+ Array.from({ length: 8 }, (_, i) => ({
+ fields: { Name: `Linked ${i + 1}`, Amount: (i + 1) * 10, Category: ['X', 'Y', 'Z'][i % 3] },
+ }))
+ );
+ const linkIds = linkedRecs.map((r) => r.id);
+
+ const img =
+ 'https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/280px-PNG_transparency_demonstration_1.png';
+ const selectNames = ['Alpha', 'Beta', 'Gamma', 'Delta 🚀', 'Has, comma'];
+ const allRecords = Array.from({ length: 20 }, (_, i) => {
+ if (i === 0) return { fields: {} }; // an all-empty row
+ return {
+ fields: {
+ Name: `Row ${i}${i % 5 === 0 ? ' 🚀 unicode ' + 'x'.repeat(80) : ''}${i === 7 ? ' nul byte' : ''}`,
+ 'Long text': `Notes ${i}\nsecond line${i === 7 ? ' ' : ''}`,
+ Email: `user${i}@example.com`,
+ URL: 'https://example.com/p?q=1',
+ Phone: '+1 555 0100',
+ Barcode: { text: `01234${i}` },
+ Int: i % 2 === 0 ? i : -i, // include negatives + zero
+ 'Decimal 1': i + 0.25,
+ 'Decimal 4': i + 0.0001,
+ 'Percent 0': (i % 100) / 100,
+ 'Percent 2': (i * 3) / 100,
+ 'Currency USD': i * 9.99,
+ 'Currency EUR': i * 100,
+ 'Dur h:mm': i * 3661, // seconds
+ 'Rate star 5': (i % 5) + 1, // 1..5 (rating rejects 0)
+ 'Rate heart 10': (i % 10) + 1, // 1..10
+ 'Single select': selectNames[i % selectNames.length],
+ 'Multi select': [['One'], ['One', 'Two'], ['Two', 'Three', 'Four']][i % 3],
+ 'Check green': i % 2 === 0, // false rows test boolean omission
+ 'Date iso': `2026-0${(i % 9) + 1}-15`,
+ 'DT utc 24': `2026-03-15T0${i % 9}:30:00.000Z`,
+ Attachments:
+ i % 6 === 0 ? [{ url: img }, { url: img }] : i % 3 === 0 ? [{ url: img }] : undefined,
+ 'Link to B': linkIds.slice(0, (i % 3) + 1),
+ 'Link to B 2': linkIds.slice(3, 5),
+ },
+ };
+ });
+ const aRecs = await createRecords(A, allRecords);
+ const aIds = aRecs.map((r) => r.id);
+
+ // self links (A->A) and the back link (B->A), now that A ids exist
+ if (fieldIds['Self link'])
+ await patchRecords(
+ A,
+ aIds.slice(1, 6).map((id, i) => ({
+ id,
+ fields: { 'Self link': aIds.slice(0, (i % 2) + 1) },
+ }))
+ );
+ if (fieldIds['Link to A'])
+ await patchRecords(
+ B,
+ linkIds.slice(0, 5).map((id, i) => ({ id, fields: { 'Link to A': [aIds[i % aIds.length]] } }))
+ );
+
+ await createRecords(
+ C,
+ Array.from({ length: 4 }, (_, i) => ({
+ fields: { Title: `Self ${i + 1}`, 'C to A': [aIds[i % aIds.length]] },
+ }))
+ );
+
+ console.log(
+ `\n✓ built "${BASE_NAME}": ${aRecs.length} rows in All Fields, ${linkIds.length} in Linked, ${Object.keys(fieldIds).length} fields created.`
+ );
+ if (failedFields.length) {
+ console.log(`\n⚠ ${failedFields.length} field(s) the API could not create — add in the UI:`);
+ for (const f of failedFields) console.log(` - ${f}`);
+ }
+ console.log('\nStill add manually in the Airtable UI (API cannot create these):');
+ console.log(
+ ' • Formulas: numeric, text concat, date (DATEADD/DATETIME_FORMAT), logical (IF), a deliberately invalid one, and one using "^" (incompatible)'
+ );
+ console.log(
+ ' • Toggle single-link on "Link to B 2" and "Link to A" (API cannot set prefersSingleRecordLink) — needed for single links + the one-to-many ManyOne case'
+ );
+ console.log(
+ ' • single->many-to-many relax test: toggle a link to single, then link several records to the SAME target so its single-side data holds multiple values'
+ );
+ console.log(
+ ' • On "Link to B": Rollup SUM/MAX/AVERAGE/COUNTALL/ARRAYJOIN/ARRAYUNIQUE, Count, Lookup (number + text + date)'
+ );
+ console.log(' • aiText field (if testing AI mapping)');
+ console.log(' • Rename one "Single select"/"Multi select" option to blank (empty name)');
+ console.log(
+ ' • Views: Kanban×2 (by select + by collaborator), Calendar, Gallery, Timeline, List, plus a Grid with filter+sort+group+hidden fields+row height'
+ );
+ console.log(' • In "Self": make the primary "Title" a formula');
+ console.log(' • Turn on Share → Share to web to test view-config extraction');
+ console.log(`\nBase id: ${baseId}`);
+};
+
+main().catch((e) => {
+ console.error('\nFATAL:', e.message);
+ process.exit(1);
+});
diff --git a/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-field-coverage.mjs b/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-field-coverage.mjs
new file mode 100644
index 0000000000..ebca89726e
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-field-coverage.mjs
@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+/**
+ * Scans every Airtable base the PAT can read and tallies field-type and link
+ * relationship coverage — to confirm the real-base corpus already exercises the
+ * computed / lookup / link combinations the meta API cannot create.
+ *
+ * Usage: AIRTABLE_PAT=patXXX node src/features/airtable-import/test-scripts/airtable-field-coverage.mjs
+ */
+const PAT = process.env.AIRTABLE_PAT;
+const API = 'https://api.airtable.com/v0';
+if (!PAT) {
+ console.error('Set AIRTABLE_PAT (schema.bases:read).');
+ process.exit(1);
+}
+const h = { Authorization: `Bearer ${PAT}` };
+const get = async (p) => (await fetch(`${API}${p}`, { headers: h })).json();
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+const main = async () => {
+ const { bases = [] } = await get('/meta/bases');
+ const typeBases = new Map(); // fieldType -> Set(baseName)
+ const typeCount = new Map(); // fieldType -> total field count
+ const links = {
+ single: new Set(),
+ multi: 0,
+ self: new Set(),
+ oneWay: new Set(),
+ sameTablePair: new Set(),
+ };
+
+ for (const base of bases) {
+ let schema;
+ try {
+ schema = await get(`/meta/bases/${base.id}/tables`);
+ } catch {
+ continue;
+ }
+ await sleep(120);
+ for (const table of schema.tables ?? []) {
+ const linkTargets = new Map(); // linkedTableId -> count (for same-table-pair detection)
+ for (const f of table.fields ?? []) {
+ typeCount.set(f.type, (typeCount.get(f.type) ?? 0) + 1);
+ if (!typeBases.has(f.type)) typeBases.set(f.type, new Set());
+ typeBases.get(f.type).add(base.name);
+ if (f.type === 'multipleRecordLinks') {
+ const o = f.options ?? {};
+ if (o.prefersSingleRecordLink) links.single.add(base.name);
+ else links.multi++;
+ if (o.linkedTableId === table.id) links.self.add(base.name);
+ if (!o.inverseLinkFieldId) links.oneWay.add(base.name);
+ linkTargets.set(o.linkedTableId, (linkTargets.get(o.linkedTableId) ?? 0) + 1);
+ }
+ }
+ for (const n of linkTargets.values()) if (n > 1) links.sameTablePair.add(base.name);
+ }
+ }
+
+ const tricky = [
+ 'multipleLookupValues',
+ 'rollup',
+ 'count',
+ 'formula',
+ 'createdTime',
+ 'lastModifiedTime',
+ 'createdBy',
+ 'lastModifiedBy',
+ 'autoNumber',
+ 'button',
+ 'aiText',
+ 'multipleAttachments',
+ 'barcode',
+ 'multipleRecordLinks',
+ ];
+ console.log(
+ `Scanned ${bases.length} bases.\n=== Field-type coverage (computed / hard-to-create) ===`
+ );
+ for (const t of tricky) {
+ const b = typeBases.get(t);
+ if (b)
+ console.log(
+ ` ${t.padEnd(22)} ${String(typeCount.get(t)).padStart(4)} fields in ${b.size} bases`
+ );
+ else console.log(` ${t.padEnd(22)} — not present in any base`);
+ }
+ console.log('\n=== All field types seen ===');
+ console.log(' ' + [...typeCount.keys()].sort().join(', '));
+ console.log('\n=== Link relationship combinations ===');
+ console.log(` single-record links (prefersSingle): ${links.single.size} bases`);
+ console.log(` multi-record links: ${links.multi} fields`);
+ console.log(` self links (table -> itself): ${links.self.size} bases`);
+ console.log(` one-way links (no inverse): ${links.oneWay.size} bases`);
+ console.log(` multiple links between same 2 tables: ${links.sameTablePair.size} bases`);
+};
+
+main().catch((e) => console.error('FATAL:', e.message));
diff --git a/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-import-smoke.mjs b/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-import-smoke.mjs
new file mode 100644
index 0000000000..ca3537cd66
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-import-smoke.mjs
@@ -0,0 +1,184 @@
+#!/usr/bin/env node
+/**
+ * Airtable import smoke test against a LOCAL backend.
+ *
+ * Signs in, resolves the connected Airtable integration, lists the accessible
+ * Airtable bases (copy the templates you want to cover into your account first),
+ * imports each one through the real SSE endpoint, and reports per-base status +
+ * the degradation issues. Re-runnable for local regression testing.
+ *
+ * Usage:
+ * node src/features/airtable-import/test-scripts/airtable-import-smoke.mjs [nameFilter] [--keep]
+ * BACKEND=http://localhost:3603/api EMAIL=... PASSWORD=... node src/features/airtable-import/test-scripts/airtable-import-smoke.mjs
+ *
+ * nameFilter case-insensitive substring to limit which bases run (optional)
+ * --keep do not delete any created base (default: delete clean ones,
+ * keep the ones that failed or reported issues for inspection)
+ */
+const BACKEND = process.env.BACKEND ?? 'http://localhost:3603/api';
+const EMAIL = process.env.EMAIL;
+const PASSWORD = process.env.PASSWORD;
+if (!EMAIL || !PASSWORD) {
+ console.error('Set EMAIL and PASSWORD (a local Teable account) before running.');
+ process.exit(1);
+}
+const args = process.argv.slice(2);
+const keepAll = args.includes('--keep');
+const deleteAll = args.includes('--delete-all');
+const nameFilter = args.find((a) => !a.startsWith('--'))?.toLowerCase();
+
+let cookie = '';
+const headers = () => ({ 'Content-Type': 'application/json', ...(cookie ? { cookie } : {}) });
+const captureCookie = (res) => {
+ const set = res.headers.getSetCookie?.() ?? [];
+ if (set.length) cookie = set.map((c) => c.split(';')[0]).join('; ');
+};
+
+const post = async (path, body) => {
+ const res = await fetch(`${BACKEND}${path}`, {
+ method: 'POST',
+ headers: headers(),
+ body: JSON.stringify(body),
+ });
+ captureCookie(res);
+ return res;
+};
+const get = async (path) => {
+ const res = await fetch(`${BACKEND}${path}`, { headers: headers() });
+ captureCookie(res);
+ return res;
+};
+
+const signin = async () => {
+ const res = await post('/auth/signin', { email: EMAIL, password: PASSWORD });
+ if (!res.ok) throw new Error(`signin failed: ${res.status} ${await res.text()}`);
+ console.log(`✓ signed in as ${EMAIL}`);
+};
+
+const getIntegrationId = async () => {
+ const res = await get('/user-integrations?provider=airtable');
+ if (!res.ok) throw new Error(`integration list failed: ${res.status} ${await res.text()}`);
+ const { integrations = [] } = await res.json();
+ const integration = integrations.find((i) => i.hasSecret) ?? integrations[0];
+ if (!integration) throw new Error('no connected Airtable integration found');
+ console.log(`✓ integration: ${integration.id}`);
+ return integration.id;
+};
+
+const getSpaceId = async () => {
+ if (process.env.SPACE_ID) return process.env.SPACE_ID;
+ const res = await get('/space');
+ if (!res.ok) throw new Error(`space list failed: ${res.status} ${await res.text()}`);
+ const spaces = await res.json();
+ if (!spaces.length) throw new Error('no space found');
+ console.log(`✓ space: ${spaces[0].id} (${spaces[0].name})`);
+ return spaces[0].id;
+};
+
+const listBases = async (integrationId) => {
+ const res = await post('/base/import-airtable/analyze', { integrationId });
+ if (!res.ok) throw new Error(`analyze failed: ${res.status} ${await res.text()}`);
+ const { bases = [] } = await res.json();
+ return bases;
+};
+
+// Reads the SSE import stream to completion, returning the final vo or throwing.
+const runImport = async (payload) => {
+ const res = await fetch(`${BACKEND}/base/import-airtable/stream`, {
+ method: 'POST',
+ headers: { ...headers(), Accept: 'text/event-stream' },
+ body: JSON.stringify(payload),
+ });
+ if (!res.ok) throw new Error(`HTTP ${res.status}: ${await res.text()}`);
+ const reader = res.body.getReader();
+ const decoder = new TextDecoder();
+ let buffer = '';
+ let result;
+ // eslint-disable-next-line no-constant-condition
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split('\n');
+ buffer = lines.pop() ?? '';
+ for (const line of lines) {
+ if (!line.startsWith('data: ')) continue;
+ const json = line.slice(6).trim();
+ if (!json) continue;
+ const event = JSON.parse(json);
+ if (event.type === 'done') result = event.data;
+ else if (event.type === 'error') throw new Error(event.message);
+ }
+ }
+ if (!result) throw new Error('stream ended without a result');
+ return result;
+};
+
+const deleteBase = async (baseId) => {
+ const res = await fetch(`${BACKEND}/base/${baseId}`, { method: 'DELETE', headers: headers() });
+ return res.ok;
+};
+
+const main = async () => {
+ await signin();
+ const [integrationId, spaceId] = [await getIntegrationId(), await getSpaceId()];
+ let bases = await listBases(integrationId);
+ if (nameFilter) bases = bases.filter((b) => b.name.toLowerCase().includes(nameFilter));
+ console.log(
+ `\nRunning ${bases.length} base(s)${nameFilter ? ` matching "${nameFilter}"` : ''}:\n`
+ );
+
+ const report = [];
+ for (const base of bases) {
+ const started = Date.now();
+ const row = { name: base.name, status: '', tables: 0, issues: 0, kept: '', detail: '' };
+ try {
+ const vo = await runImport({
+ spaceId,
+ integrationId,
+ airtableBaseId: base.id,
+ baseName: `[smoke] ${base.name}`,
+ importRecords: true,
+ importAttachments: !process.env.NO_ATTACH,
+ });
+ row.status = '✓ ok';
+ row.tables = Object.keys(vo.tableIdMap).length;
+ row.issues = vo.issues.length;
+ const clean = vo.issues.length === 0;
+ if (deleteAll || (!keepAll && clean)) {
+ await deleteBase(vo.base.id);
+ row.kept = '(deleted)';
+ } else {
+ row.kept = vo.base.id;
+ }
+ // surface the distinct issue reasons for quick triage
+ const reasons = [...new Set(vo.issues.map((i) => `${i.code}:${i.reason ?? i.toType ?? ''}`))];
+ row.detail = reasons.slice(0, 6).join(' | ');
+ } catch (e) {
+ row.status = '✗ FAIL';
+ row.detail = e instanceof Error ? e.message : String(e);
+ }
+ row.ms = Date.now() - started;
+ report.push(row);
+ console.log(
+ `${row.status.padEnd(7)} ${base.name.padEnd(28)} ${String(row.tables).padStart(2)}t ` +
+ `${String(row.issues).padStart(3)} issues ${String(row.ms).padStart(6)}ms ${row.kept}`
+ );
+ if (row.detail) console.log(` ${row.detail}`);
+ }
+
+ const failed = report.filter((r) => r.status.includes('FAIL'));
+ console.log(
+ `\n==== ${report.length} base(s): ${report.length - failed.length} ok, ${failed.length} failed ====`
+ );
+ if (failed.length) {
+ console.log('FAILURES:');
+ for (const f of failed) console.log(` ✗ ${f.name}: ${f.detail}`);
+ process.exitCode = 1;
+ }
+};
+
+main().catch((e) => {
+ console.error('\nFATAL:', e instanceof Error ? e.message : e);
+ process.exitCode = 1;
+});
diff --git a/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-verify-share.mjs b/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-verify-share.mjs
new file mode 100644
index 0000000000..33ef9149a2
--- /dev/null
+++ b/apps/nestjs-backend/src/features/airtable-import/test-scripts/airtable-verify-share.mjs
@@ -0,0 +1,78 @@
+/**
+ * End-to-end check of the view-config share path against the kitchen-sink test
+ * base. Mirrors src/.../airtable-share.client.ts (resolveShare + fetchViewConfig)
+ * in plain JS and prints the filters / sorts / groupLevels / metadata each view
+ * exposes, to confirm a share link actually serves importable view config.
+ *
+ * Usage: node src/features/airtable-import/test-scripts/airtable-verify-share.mjs
+ * (or SHARE_LINK / VIEW_IDS env)
+ */
+const LINK = process.argv[2] ?? process.env.SHARE_LINK;
+const VIEW_IDS = (process.argv[3] ?? process.env.VIEW_IDS ?? '')
+ .split(',')
+ .map((v) => v.trim())
+ .filter(Boolean);
+if (!LINK || !VIEW_IDS.length) {
+ console.error(
+ 'Usage: airtable-verify-share.mjs (or SHARE_LINK / VIEW_IDS env).'
+ );
+ process.exit(1);
+}
+const UA =
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
+
+const main = async () => {
+ const pageRes = await fetch(LINK, {
+ headers: { 'User-Agent': UA, Accept: 'text/html' },
+ redirect: 'manual',
+ });
+ if (pageRes.status >= 300 && pageRes.status < 400) {
+ throw new Error(`not public (redirect to ${pageRes.headers.get('location')})`);
+ }
+ const cookie = (pageRes.headers.getSetCookie?.() ?? []).map((e) => e.split(';')[0]).join('; ');
+ const html = await pageRes.text();
+ const requestId = html.match(/requestId: "(.*?)",/)?.[1];
+ const initData = JSON.parse(html.match(/window\.initData = (.*?);\n/)?.[1] ?? '{}');
+ const { sharedApplicationId: appId, accessPolicy, pageLoadId, codeVersion } = initData;
+ console.log(`resolved: appId=${appId} codeVersion=${String(codeVersion).slice(0, 8)}…`);
+
+ const headers = {
+ 'User-Agent': UA,
+ Accept: 'application/json',
+ 'x-airtable-application-id': appId,
+ 'x-airtable-inter-service-client': 'webClient',
+ 'x-airtable-inter-service-client-code-version': codeVersion,
+ 'x-airtable-page-load-id': pageLoadId,
+ 'X-Requested-With': 'XMLHttpRequest',
+ 'x-time-zone': 'UTC',
+ 'x-user-locale': 'en',
+ cookie,
+ };
+ for (const viewId of VIEW_IDS) {
+ const params = new URLSearchParams({
+ stringifiedObjectParams: JSON.stringify({
+ mayOnlyIncludeRowAndCellDataForIncludedViews: true,
+ mayExcludeCellDataForLargeViews: true,
+ allowMsgpackOfResult: false,
+ }),
+ requestId,
+ accessPolicy,
+ });
+ const res = await fetch(`https://airtable.com/v0.3/view/${viewId}/readData?${params}`, {
+ headers,
+ });
+ const d = (await res.json()).data ?? {};
+ console.log(`\n# ${viewId} — ${res.status}`);
+ console.log(' filters: ', JSON.stringify(d.filters));
+ console.log(' sorts: ', JSON.stringify(d.lastSortsApplied?.sortSet));
+ console.log(' groupLevels:', JSON.stringify(d.groupLevels));
+ console.log(' metadata: ', d.metadata ? Object.keys(d.metadata).join(', ') : null);
+ await new Promise((r) => setTimeout(r, 200));
+ }
+ console.log('\n✓ share view-config read OK');
+};
+
+main().catch((e) => {
+ console.error('FAIL:', e?.message ?? e);
+ process.exit(1);
+});
diff --git a/apps/nestjs-backend/src/features/attachments/attachments.service.ts b/apps/nestjs-backend/src/features/attachments/attachments.service.ts
index d303e8231e..240141b21d 100644
--- a/apps/nestjs-backend/src/features/attachments/attachments.service.ts
+++ b/apps/nestjs-backend/src/features/attachments/attachments.service.ts
@@ -286,6 +286,31 @@ export class AttachmentsService {
return await this.notifyToAttachmentItem(token, filename);
}
+ /**
+ * Streams an already-open file stream of a known size straight into
+ * storage — no temp file, works the same for local, S3 and MinIO backends.
+ */
+ async uploadFromStream(
+ stream: Readable,
+ params: { filename: string; contentType: string; contentLength: number },
+ uploadType: UploadType = UploadType.Table
+ ): Promise {
+ const MAX_FILE_SIZE = this.thresholdConfig.maxOpenapiAttachmentUploadSize;
+ const { filename, contentType, contentLength } = params;
+ if (contentLength > MAX_FILE_SIZE) {
+ this.throwFileSizeExceeded(MAX_FILE_SIZE);
+ }
+
+ const { token, url } = await this.signature({
+ type: uploadType,
+ contentLength,
+ contentType,
+ internal: true,
+ });
+ await this.uploadStreamToStorage(url, stream, contentType, contentLength);
+ return await this.notifyToAttachmentItem(token, filename);
+ }
+
async uploadFromUrl(
fileUrl: string,
uploadType: UploadType = UploadType.Table
diff --git a/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts b/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts
index 8ade9c4780..14743f3633 100644
--- a/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts
+++ b/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts
@@ -2,6 +2,11 @@ import { SetMetadata } from '@nestjs/common';
import type { Action } from '@teable/core';
export const PERMISSIONS_KEY = 'permissions';
+export const ANY_PERMISSIONS_KEY = 'anyPermissions';
// eslint-disable-next-line @typescript-eslint/naming-convention
export const Permissions = (...permissions: Action[]) => SetMetadata(PERMISSIONS_KEY, permissions);
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export const AnyPermissions = (...permissionGroups: Action[][]) =>
+ SetMetadata(ANY_PERMISSIONS_KEY, permissionGroups);
diff --git a/apps/nestjs-backend/src/features/auth/guard/permission.guard.spec.ts b/apps/nestjs-backend/src/features/auth/guard/permission.guard.spec.ts
new file mode 100644
index 0000000000..9c13202fb8
--- /dev/null
+++ b/apps/nestjs-backend/src/features/auth/guard/permission.guard.spec.ts
@@ -0,0 +1,125 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import type { ExecutionContext } from '@nestjs/common';
+import type { Reflector } from '@nestjs/core';
+import { HttpErrorCode, type Action } from '@teable/core';
+import type { ClsService } from 'nestjs-cls';
+import { CustomHttpException } from '../../../custom.exception';
+import type { IClsStore } from '../../../types/cls';
+import type { PermissionService } from '../permission.service';
+import { PermissionGuard } from './permission.guard';
+
+vi.mock('../permission.service', () => ({
+ PermissionService: class PermissionService {},
+}));
+
+const tableId = 'tblxxxxxxxxxxxx';
+const tableUpdatePermissions: Action[] = ['table|update'];
+const instanceUpdatePermissions: Action[] = ['instance|update'];
+
+const createContext = (): ExecutionContext =>
+ ({
+ getHandler: vi.fn(),
+ getClass: vi.fn(),
+ switchToHttp: () => ({
+ getRequest: () => ({
+ params: { tableId },
+ headers: {},
+ }),
+ }),
+ }) as unknown as ExecutionContext;
+
+const createForbiddenError = (permissions: Action[]) =>
+ new CustomHttpException(
+ `not allowed to operate ${permissions.join(', ')} on ${tableId}`,
+ HttpErrorCode.RESTRICTED_RESOURCE
+ );
+
+describe('PermissionGuard', () => {
+ const createGuard = ({
+ primaryPermissions,
+ anyPermissions,
+ isAdmin = false,
+ validPermissions,
+ }: {
+ primaryPermissions: Action[];
+ anyPermissions?: Action[][];
+ isAdmin?: boolean;
+ validPermissions: PermissionService['validPermissions'];
+ }) => {
+ const reflector = {
+ getAllAndOverride: vi.fn((key: string) => {
+ if (key === 'permissions') {
+ return primaryPermissions;
+ }
+ if (key === 'anyPermissions') {
+ return anyPermissions;
+ }
+ return undefined;
+ }),
+ } as unknown as Reflector;
+ const cls = {
+ get: vi.fn((key: string) => {
+ if (key === 'user.id') {
+ return 'usrxxxxxxxxxxxx';
+ }
+ if (key === 'user.isAdmin') {
+ return isAdmin;
+ }
+ return undefined;
+ }),
+ set: vi.fn(),
+ } as unknown as ClsService;
+ const permissionService = {
+ validPermissions,
+ } as unknown as PermissionService;
+
+ return {
+ guard: new PermissionGuard(reflector, cls, permissionService),
+ cls,
+ permissionService,
+ };
+ };
+
+ it('keeps table update as the primary permission for alternative permission routes', async () => {
+ const validPermissions = vi.fn().mockResolvedValue(tableUpdatePermissions);
+ const { guard, cls } = createGuard({
+ primaryPermissions: tableUpdatePermissions,
+ anyPermissions: [instanceUpdatePermissions],
+ validPermissions,
+ });
+
+ await expect(guard.canActivate(createContext())).resolves.toBe(true);
+
+ expect(validPermissions).toHaveBeenCalledWith(tableId, tableUpdatePermissions, undefined);
+ expect(cls.get).not.toHaveBeenCalledWith('user.isAdmin');
+ });
+
+ it('allows instance admins through alternative permissions when table update is unavailable', async () => {
+ const validPermissions = vi
+ .fn()
+ .mockRejectedValue(createForbiddenError(tableUpdatePermissions));
+ const { guard } = createGuard({
+ primaryPermissions: tableUpdatePermissions,
+ anyPermissions: [instanceUpdatePermissions],
+ isAdmin: true,
+ validPermissions,
+ });
+
+ await expect(guard.canActivate(createContext())).resolves.toBe(true);
+ });
+
+ it('still rejects users without table update or instance update', async () => {
+ const validPermissions = vi
+ .fn()
+ .mockRejectedValue(createForbiddenError(tableUpdatePermissions));
+ const { guard } = createGuard({
+ primaryPermissions: tableUpdatePermissions,
+ anyPermissions: [instanceUpdatePermissions],
+ validPermissions,
+ });
+
+ await expect(guard.canActivate(createContext())).rejects.toThrow(
+ `not allowed to operate table|update on ${tableId}`
+ );
+ });
+});
diff --git a/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts b/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts
index c5f52475f8..42cb61f1ab 100644
--- a/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts
+++ b/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts
@@ -8,7 +8,7 @@ import { CustomHttpException } from '../../../custom.exception';
import type { IClsStore } from '../../../types/cls';
import { AllowAnonymousType, IS_ALLOW_ANONYMOUS } from '../decorators/allow-anonymous.decorator';
import { IS_DISABLED_PERMISSION } from '../decorators/disabled-permission.decorator';
-import { PERMISSIONS_KEY } from '../decorators/permissions.decorator';
+import { ANY_PERMISSIONS_KEY, PERMISSIONS_KEY } from '../decorators/permissions.decorator';
import { IS_PUBLIC_KEY } from '../decorators/public.decorator';
import type { IResourceMeta } from '../decorators/resource_meta.decorator';
import { RESOURCE_META } from '../decorators/resource_meta.decorator';
@@ -323,6 +323,10 @@ export class PermissionGuard {
context.getHandler(),
context.getClass(),
]);
+ const anyPermissions = this.reflector.getAllAndOverride(
+ ANY_PERMISSIONS_KEY,
+ [context.getHandler(), context.getClass()]
+ );
const resourceId = this.getResourceId(context) || this.defaultResourceId(context);
const accessTokenId = this.cls.get('accessTokenId');
if (accessTokenId && !permissions?.length) {
@@ -337,6 +341,24 @@ export class PermissionGuard {
if (!permissions?.length) {
return true;
}
+ if (anyPermissions?.length) {
+ try {
+ return await this.checkPermissions(resourceId, permissions);
+ } catch (error) {
+ for (const permissionGroup of anyPermissions) {
+ try {
+ return await this.checkPermissions(resourceId, permissionGroup);
+ } catch {
+ // Try the next alternative and preserve the primary error if all alternatives fail.
+ }
+ }
+ throw error;
+ }
+ }
+ return await this.checkPermissions(resourceId, permissions);
+ }
+
+ private async checkPermissions(resourceId: string | undefined, permissions: Action[]) {
// instance permission check
if (permissions?.includes('instance|update')) {
return this.instancePermissionChecker('instance|update');
diff --git a/apps/nestjs-backend/src/features/base-node/base-node.service.ts b/apps/nestjs-backend/src/features/base-node/base-node.service.ts
index 1f533440f8..1cd06dc9fc 100644
--- a/apps/nestjs-backend/src/features/base-node/base-node.service.ts
+++ b/apps/nestjs-backend/src/features/base-node/base-node.service.ts
@@ -20,6 +20,7 @@ import type {
IBaseNodePresenceDeletePayload,
IBaseNodePresenceUpdatePayload,
IBaseNodeTableResourceMeta,
+ IUserInfoVo,
V2Feature,
} from '@teable/openapi';
import { BaseNodeResourceType } from '@teable/openapi';
@@ -56,10 +57,26 @@ type IBaseNodeEntry = {
resourceType: string;
resourceId: string;
order: number;
+ createdBy?: string | null;
+ createdTime?: Date | string | null;
+ lastModifiedBy?: string | null;
+ lastModifiedTime?: Date | string | null;
children: { id: string; order: number }[];
parent: { id: string } | null;
};
+type IBaseNodeResourceInput = Omit<
+ IBaseNodeResourceMeta,
+ 'createdByUser' | 'lastModifiedByUser' | 'createdTime' | 'lastModifiedTime'
+> & {
+ id: string;
+ type?: BaseNodeResourceType;
+ createdBy?: string | null;
+ createdTime?: Date | string | null;
+ lastModifiedBy?: string | null;
+ lastModifiedTime?: Date | string | null;
+};
+
@Injectable()
export class BaseNodeService {
private readonly logger = new Logger(BaseNodeService.name);
@@ -115,6 +132,10 @@ export class BaseNodeService {
resourceType: true,
resourceId: true,
order: true,
+ createdBy: true,
+ createdTime: true,
+ lastModifiedBy: true,
+ lastModifiedTime: true,
children: {
select: { id: true, order: true },
orderBy: { order: 'asc' as const },
@@ -190,68 +211,115 @@ export class BaseNodeService {
private async entry2vo(
entry: IBaseNodeEntry,
- resource?: IBaseNodeResourceMeta
+ resource?: IBaseNodeResourceInput,
+ userMap?: Map
): Promise {
- const resourceMeta =
- resource ||
+ const baseNodeEntry = omit(entry, [
+ 'createdBy',
+ 'createdTime',
+ 'lastModifiedBy',
+ 'lastModifiedTime',
+ ]) as Omit;
+ const resourceType = baseNodeEntry.resourceType as BaseNodeResourceType;
+ const rawResource: IBaseNodeResourceInput =
+ resource ??
(
- await this.getNodeResource(entry.baseId, entry.resourceType as BaseNodeResourceType, [
- entry.resourceId,
- ])
+ await this.getNodeResource(baseNodeEntry.baseId, resourceType, [baseNodeEntry.resourceId])
)[0];
- const resourceMetaWithoutId = resource ? resource : omit(resourceMeta, 'id');
+ const { id, type, createdBy, createdTime, lastModifiedBy, lastModifiedTime, ...resourceMeta } =
+ rawResource;
+ const map = userMap ?? (await this.getUserMap([createdBy, lastModifiedBy]));
+ const resolveUser = (userId?: string | null) => (userId ? map.get(userId) ?? null : null);
+ const formatDate = (value: unknown) =>
+ value instanceof Date ? value.toISOString() : typeof value === 'string' ? value : null;
+ const resourceMetaWithAudit = {
+ ...resourceMeta,
+ createdTime: formatDate(createdTime),
+ createdByUser: resolveUser(createdBy),
+ lastModifiedTime: formatDate(lastModifiedTime),
+ lastModifiedByUser: resolveUser(lastModifiedBy),
+ } as IBaseNodeResourceMeta;
const defaultUrl = this.generateDefaultUrl(
- entry.baseId,
- entry.resourceType as BaseNodeResourceType,
- entry.resourceId,
- resourceMetaWithoutId
+ baseNodeEntry.baseId,
+ resourceType,
+ baseNodeEntry.resourceId,
+ resourceMetaWithAudit
);
return {
- ...entry,
- resourceType: entry.resourceType as BaseNodeResourceType,
- resourceMeta: resourceMetaWithoutId,
+ ...baseNodeEntry,
+ resourceType,
+ resourceMeta: resourceMetaWithAudit,
defaultUrl,
};
}
- protected getTableResources(baseId: string, ids?: string[]) {
- return this.prismaService.tableMeta.findMany({
+ private async getUserMap(
+ userIds: Array
+ ): Promise> {
+ const ids = [...new Set(userIds.filter(Boolean) as string[])];
+ if (ids.length === 0) {
+ return new Map();
+ }
+
+ const users = await this.prismaService.user.findMany({
+ where: { id: { in: ids } },
+ select: {
+ id: true,
+ name: true,
+ email: true,
+ avatar: true,
+ },
+ });
+
+ return new Map(users.map((user) => [user.id, user]));
+ }
+
+ protected async getTableResources(baseId: string, ids?: string[]) {
+ return await this.prismaService.tableMeta.findMany({
where: { baseId, id: { in: ids ? ids : undefined }, deletedTime: null },
select: {
id: true,
name: true,
icon: true,
+ createdBy: true,
+ createdTime: true,
+ lastModifiedBy: true,
+ lastModifiedTime: true,
},
});
}
- protected getDashboardResources(baseId: string, ids?: string[]) {
- return this.prismaService.dashboard.findMany({
+ protected async getDashboardResources(baseId: string, ids?: string[]) {
+ return await this.prismaService.dashboard.findMany({
where: { baseId, id: { in: ids ? ids : undefined } },
select: {
id: true,
name: true,
+ createdBy: true,
+ createdTime: true,
+ lastModifiedBy: true,
+ lastModifiedTime: true,
},
});
}
- protected getFolderResources(baseId: string, ids?: string[]) {
- return this.prismaService.baseNodeFolder.findMany({
+ protected async getFolderResources(baseId: string, ids?: string[]) {
+ return await this.prismaService.baseNodeFolder.findMany({
where: { baseId, id: { in: ids ? ids : undefined } },
select: {
id: true,
name: true,
+ createdBy: true,
+ createdTime: true,
+ lastModifiedBy: true,
+ lastModifiedTime: true,
},
});
}
- protected async getNodeResource(
- baseId: string,
- type: BaseNodeResourceType,
- ids?: string[]
- ): Promise {
+ protected async getNodeResource(baseId: string, type: BaseNodeResourceType, ids?: string[]) {
switch (type) {
case BaseNodeResourceType.Folder:
return this.getFolderResources(baseId, ids);
@@ -290,6 +358,9 @@ export class BaseNodeService {
list.map((r) => ({ ...r, type: resourceTypes[index] }))
);
+ const userMap = await this.getUserMap(
+ resources.flatMap((resource) => [resource.createdBy, resource.lastModifiedBy])
+ );
const resourceMap = keyBy(resources, (r) => `${r.type}_${r.id}`);
const resourceKeys = new Set(resources.map((r) => `${r.type}_${r.id}`));
@@ -309,23 +380,13 @@ export class BaseNodeService {
);
if (toCreate.length === 0 && toDelete.length === 0 && orphans.length === 0) {
- return nodes.map((entry) => {
- const key = `${entry.resourceType}_${entry.resourceId}`;
- const resource = resourceMap[key];
- const resourceMeta = omit(resource, 'id');
- const defaultUrl = this.generateDefaultUrl(
- baseId,
- entry.resourceType as BaseNodeResourceType,
- entry.resourceId,
- resourceMeta
- );
- return {
- ...entry,
- resourceType: entry.resourceType as BaseNodeResourceType,
- resourceMeta,
- defaultUrl,
- };
- });
+ return Promise.all(
+ nodes.map((entry) => {
+ const key = `${entry.resourceType}_${entry.resourceId}`;
+ const resource = resourceMap[key];
+ return this.entry2vo(entry, resource, userMap);
+ })
+ );
}
const finalMenus = await this.prismaService.$tx(async (prisma) => {
@@ -381,7 +442,7 @@ export class BaseNodeService {
finalMenus.map(async (entry) => {
const key = `${entry.resourceType}_${entry.resourceId}`;
const resource = resourceMap[key];
- return await this.entry2vo(entry, omit(resource, 'id'));
+ return await this.entry2vo(entry, resource, userMap);
})
);
}
@@ -455,7 +516,7 @@ export class BaseNodeService {
select: this.getSelect(),
});
- const vo = await this.entry2vo(entry, omit(resource, 'id'));
+ const vo = await this.entry2vo(entry, resource);
this.presenceHandler(baseId, (presence) => {
presence.submit({
event: 'create',
@@ -604,7 +665,7 @@ export class BaseNodeService {
};
});
- const vo = await this.entry2vo(entry, omit(resource, 'id'));
+ const vo = await this.entry2vo(entry, resource);
this.presenceHandler(baseId, (presence) => {
presence.submit({
event: 'create',
diff --git a/apps/nestjs-backend/src/features/base-share/base-share.service.ts b/apps/nestjs-backend/src/features/base-share/base-share.service.ts
index 2d49402469..bb623580b8 100644
--- a/apps/nestjs-backend/src/features/base-share/base-share.service.ts
+++ b/apps/nestjs-backend/src/features/base-share/base-share.service.ts
@@ -5,9 +5,12 @@ import type { ICreateBaseShareRo, IUpdateBaseShareRo, IBaseShareVo } from '@teab
import { BaseNodeResourceType } from '@teable/openapi';
import { ClsService } from 'nestjs-cls';
import { CustomHttpException } from '../../custom.exception';
+import { Events } from '../../event-emitter/events';
import { PerformanceCache, PerformanceCacheService } from '../../performance-cache';
import { generateBaseShareListCacheKey } from '../../performance-cache/generate-keys';
import type { IClsStore } from '../../types/cls';
+import { AuditScope } from '../audit/audit-scope';
+import { Audit } from '../audit/audit.decorator';
const baseShareNotFoundMessage = 'Base share not found';
const baseShareNotFoundKey = 'httpErrors.baseShare.notFound';
@@ -19,7 +22,8 @@ export class BaseShareService {
constructor(
private readonly prismaService: PrismaService,
private readonly cls: ClsService,
- private readonly performanceCacheService: PerformanceCacheService
+ private readonly performanceCacheService: PerformanceCacheService,
+ private readonly audit: AuditScope
) {}
private async invalidateBaseShareListCache(baseId: string): Promise {
@@ -75,6 +79,16 @@ export class BaseShareService {
};
}
+ @Audit({
+ action: Events.BASE_SHARE_CREATE,
+ resourceId: (baseId: string) => baseId,
+ params: (baseId: string, data: ICreateBaseShareRo) => ({
+ baseId,
+ nodeId: data.nodeId ?? null,
+ type: data.nodeId ? 'node' : 'base',
+ }),
+ emit: (result: IBaseShareVo) => ({ shareId: result.shareId, enabled: result.enabled }),
+ })
async createBaseShare(baseId: string, data: ICreateBaseShareRo): Promise {
const nodeId = data.nodeId ?? null;
@@ -154,6 +168,22 @@ export class BaseShareService {
return this.formatBaseShareVo(share);
}
+ @Audit({
+ action: Events.BASE_SHARE_UPDATE,
+ resourceId: (_baseId: string, shareId: string) => shareId,
+ params: (baseId: string, shareId: string, data: IUpdateBaseShareRo) => ({
+ baseId,
+ shareId,
+ // First-pass: store raw request body so any field change is preserved.
+ // Password value itself is never logged — controller-layer secrets stay opaque.
+ changes: { ...data, password: data.password !== undefined ? '[set]' : undefined },
+ }),
+ emit: (result: IBaseShareVo) => ({
+ nodeId: result.nodeId,
+ type: result.nodeId ? 'node' : 'base',
+ enabled: result.enabled,
+ }),
+ })
async updateBaseShare(
baseId: string,
shareId: string,
@@ -199,6 +229,12 @@ export class BaseShareService {
return this.formatBaseShareVo(updated);
}
+ @Audit({
+ action: Events.BASE_SHARE_DELETE,
+ resourceId: (_baseId: string, shareId: string) => shareId,
+ params: (baseId: string, shareId: string) => ({ baseId, shareId }),
+ emit: true,
+ })
async deleteBaseShare(baseId: string, shareId: string): Promise {
const share = await this.prismaService.baseShare.findFirst({
where: { baseId, shareId, enabled: true },
@@ -222,6 +258,12 @@ export class BaseShareService {
await this.invalidateBaseShareListCache(baseId);
}
+ @Audit({
+ action: Events.BASE_SHARE_REFRESH,
+ resourceId: (_baseId: string, shareId: string) => shareId,
+ params: (baseId: string, shareId: string) => ({ baseId, oldShareId: shareId }),
+ emit: (result: IBaseShareVo) => ({ newShareId: result.shareId }),
+ })
async refreshBaseShareId(baseId: string, shareId: string): Promise {
const share = await this.prismaService.baseShare.findFirst({
where: { baseId, shareId, enabled: true },
diff --git a/apps/nestjs-backend/src/features/base-sql-executor/allowed-functions.ts b/apps/nestjs-backend/src/features/base-sql-executor/allowed-functions.ts
new file mode 100644
index 0000000000..c965202d92
--- /dev/null
+++ b/apps/nestjs-backend/src/features/base-sql-executor/allowed-functions.ts
@@ -0,0 +1,144 @@
+/**
+ * Whitelist of PostgreSQL functions allowed in user-facing sql-query API.
+ * Any function NOT in this set will be rejected at the AST validation layer.
+ *
+ * Criteria for inclusion:
+ * - IMMUTABLE or STABLE, or VOLATILE but side-effect-free (e.g. now(), timezone())
+ * - Cannot execute embedded SQL (excludes query_to_xml, ts_stat, ts_rewrite, etc.)
+ * - Cannot modify database state (excludes setval, lo_create, set_config, etc.)
+ * - Cannot acquire locks or send notifications (excludes pg_advisory_lock, pg_notify, etc.)
+ * - Cannot generate unbounded rows from nothing (excludes generate_series)
+ */
+export const allowedFunctions = new Set([
+ // ── Aggregation ──
+ 'avg',
+ 'bool_and',
+ 'bool_or',
+ 'count',
+ 'every',
+ 'json_agg',
+ 'jsonb_agg',
+ 'max',
+ 'min',
+ 'string_agg',
+ 'sum',
+ 'array_agg',
+
+ // ── Window ──
+ 'cume_dist',
+ 'dense_rank',
+ 'first_value',
+ 'lag',
+ 'last_value',
+ 'lead',
+ 'nth_value',
+ 'ntile',
+ 'percent_rank',
+ 'rank',
+ 'row_number',
+
+ // ── Math ──
+ 'abs',
+ 'ceil',
+ 'ceiling',
+ 'div',
+ 'floor',
+ 'greatest',
+ 'least',
+ 'mod',
+ 'power',
+ 'pow',
+ 'round',
+ 'sign',
+ 'sqrt',
+ 'trunc',
+
+ // ── String ──
+ 'ascii',
+ 'char_length',
+ 'chr',
+ 'concat',
+ 'concat_ws',
+ 'initcap',
+ 'left',
+ 'length',
+ 'lower',
+ 'lpad',
+ 'ltrim',
+ 'md5',
+ 'position',
+ 'regexp_match',
+ 'regexp_matches',
+ 'regexp_replace',
+ 'repeat',
+ 'replace',
+ 'reverse',
+ 'right',
+ 'rpad',
+ 'rtrim',
+ 'split_part',
+ 'starts_with',
+ 'strpos',
+ 'substr',
+ 'substring',
+ 'translate',
+ 'trim',
+ 'upper',
+
+ // ── Date / Time ──
+ 'age',
+ 'date_part',
+ 'date_trunc',
+ 'extract',
+ 'make_date',
+ 'make_timestamp',
+ 'now',
+ 'to_char',
+ 'to_date',
+ 'to_number',
+ 'to_timestamp',
+ 'timezone',
+
+ // ── JSON / JSONB ──
+ 'json_array_elements',
+ 'json_array_elements_text',
+ 'json_array_length',
+ 'json_build_array',
+ 'json_build_object',
+ 'json_extract_path',
+ 'json_extract_path_text',
+ 'json_typeof',
+ 'jsonb_array_elements',
+ 'jsonb_array_elements_text',
+ 'jsonb_array_length',
+ 'jsonb_build_array',
+ 'jsonb_build_object',
+ 'jsonb_each',
+ 'jsonb_each_text',
+ 'jsonb_extract_path',
+ 'jsonb_extract_path_text',
+ 'jsonb_object_keys',
+ 'jsonb_pretty',
+ 'jsonb_set',
+ 'jsonb_strip_nulls',
+ 'jsonb_typeof',
+ 'to_json',
+ 'to_jsonb',
+
+ // ── Array ──
+ 'array_append',
+ 'array_cat',
+ 'array_length',
+ 'array_position',
+ 'array_remove',
+ 'array_to_string',
+ 'string_to_array',
+ 'unnest',
+
+ // ── Conditional ──
+ 'coalesce',
+ 'nullif',
+
+ // ── Type conversion ──
+ 'cast',
+]);
diff --git a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.spec.ts b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.spec.ts
new file mode 100644
index 0000000000..5983e1c6e2
--- /dev/null
+++ b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.spec.ts
@@ -0,0 +1,90 @@
+import { ConfigService } from '@nestjs/config';
+import knex from 'knex';
+import { describe, expect, it, vi } from 'vitest';
+import { BaseSqlExecutorService } from './base-sql-executor.service';
+
+const baseId = 'bsexxx';
+const tableDbName = 'bsexxx.tblOrders';
+const sql = 'SELECT count(*) FROM "bsexxx"."tblOrders"';
+
+const createService = ({
+ prismaService,
+ databaseRouter,
+}: {
+ prismaService: unknown;
+ databaseRouter: unknown;
+}) =>
+ new BaseSqlExecutorService(
+ prismaService as never,
+ databaseRouter as never,
+ {
+ get: vi.fn().mockReturnValue('postgresql://teable:secret@localhost:5432/teable'),
+ } as unknown as ConfigService,
+ knex({ client: 'pg' }) as never,
+ { searchTimeout: 30_000 } as never
+ );
+
+const createPrismaService = () => ({
+ tableMeta: {
+ findMany: vi.fn().mockResolvedValue([{ dbTableName: tableDbName }]),
+ },
+});
+
+describe('BaseSqlExecutorService', () => {
+ it('executes BYODB sql-query without creating or setting a read-only role', async () => {
+ const prismaService = createPrismaService();
+ const transactionPrisma = {
+ $executeRawUnsafe: vi.fn().mockResolvedValue(0),
+ $queryRawUnsafe: vi.fn().mockResolvedValue([{ count: 1n }]),
+ };
+ const databaseRouter = {
+ getDataDatabaseForBase: vi.fn().mockResolvedValue({
+ isMetaFallback: false,
+ url: 'postgresql://teable:secret@byodb.example.com:5432/teable',
+ }),
+ dataPrismaExecutorForBase: vi.fn(),
+ dataPrismaTransactionForBase: vi.fn(async (_baseId: string, fn: (prisma: never) => unknown) =>
+ fn(transactionPrisma as never)
+ ),
+ };
+ const service = createService({ prismaService, databaseRouter });
+
+ await expect(service.executeQuerySql(baseId, sql)).resolves.toEqual([{ count: 1n }]);
+
+ expect(databaseRouter.getDataDatabaseForBase).toHaveBeenCalledWith(baseId);
+ expect(databaseRouter.dataPrismaExecutorForBase).not.toHaveBeenCalled();
+ expect(transactionPrisma.$executeRawUnsafe).toHaveBeenCalledWith('SET TRANSACTION READ ONLY');
+ expect(transactionPrisma.$executeRawUnsafe.mock.calls).toEqual(
+ expect.not.arrayContaining([[expect.stringContaining('SET LOCAL ROLE')]])
+ );
+ });
+
+ it('keeps using the read-only role for default data storage', async () => {
+ const prismaService = createPrismaService();
+ const rolePrisma = {
+ $queryRawUnsafe: vi.fn().mockResolvedValue([{ count: 1n }]),
+ };
+ const transactionPrisma = {
+ $executeRawUnsafe: vi.fn().mockResolvedValue(0),
+ $queryRawUnsafe: vi.fn().mockResolvedValue([{ count: 1n }]),
+ };
+ const databaseRouter = {
+ getDataDatabaseForBase: vi.fn().mockResolvedValue({
+ isMetaFallback: true,
+ url: 'postgresql://teable:secret@default.example.com:5432/teable',
+ }),
+ dataPrismaExecutorForBase: vi.fn().mockResolvedValue(rolePrisma),
+ dataPrismaTransactionForBase: vi.fn(async (_baseId: string, fn: (prisma: never) => unknown) =>
+ fn(transactionPrisma as never)
+ ),
+ };
+ const service = createService({ prismaService, databaseRouter });
+
+ await expect(service.executeQuerySql(baseId, sql)).resolves.toEqual([{ count: 1n }]);
+
+ expect(databaseRouter.dataPrismaExecutorForBase).toHaveBeenCalledWith(baseId);
+ expect(transactionPrisma.$executeRawUnsafe.mock.calls).toEqual(
+ expect.arrayContaining([[expect.stringContaining('SET LOCAL ROLE')]])
+ );
+ });
+});
diff --git a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts
index d8f21106e4..3940826edf 100644
--- a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts
+++ b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts
@@ -5,6 +5,7 @@ import { DriverClient, HttpErrorCode, parseDsn } from '@teable/core';
import { Prisma, PrismaService, getDatabaseUrl } from '@teable/db-main-prisma';
import { Knex } from 'knex';
import { InjectModel } from 'nest-knexjs';
+import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config';
import { CustomHttpException } from '../../custom.exception';
import { DatabaseRouter } from '../../global/database-router.service';
import { DATA_KNEX } from '../../global/knex';
@@ -21,7 +22,8 @@ export class BaseSqlExecutorService {
private readonly prismaService: PrismaService,
private readonly databaseRouter: DatabaseRouter,
private readonly configService: ConfigService,
- @InjectModel(DATA_KNEX) private readonly knex: Knex
+ @InjectModel(DATA_KNEX) private readonly knex: Knex,
+ @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig
) {
this.dsn = parseDsn(this.getDatabaseUrl());
this.driver = this.dsn.driver as DriverClient;
@@ -120,9 +122,13 @@ export class BaseSqlExecutorService {
return Boolean(roleExists[0].count);
}
- private async roleCheckAndCreate(baseId: string) {
+ private async roleCheckAndCreate(baseId: string): Promise {
if (this.driver !== DriverClient.Pg) {
- return;
+ return false;
+ }
+ const resolvedDataDb = await this.databaseRouter.getDataDatabaseForBase(baseId);
+ if (!resolvedDataDb.isMetaFallback) {
+ return false;
}
const roleName = this.getReadOnlyRoleName(baseId);
if (!(await this.roleExits(roleName, baseId))) {
@@ -137,30 +143,35 @@ export class BaseSqlExecutorService {
this.logger.warn(
`read only role ${roleName} already exists (concurrent creation), skipping`
);
- return;
+ return true;
}
throw error;
}
}
+ return true;
}
- private async setRole(
+ private async setLocalRole(
prisma: { $executeRawUnsafe(query: string): Promise },
baseId: string
) {
const roleName = this.getReadOnlyRoleName(baseId);
- await prisma.$executeRawUnsafe(this.knex.raw(`SET ROLE ??`, [roleName]).toQuery());
+ await prisma.$executeRawUnsafe(this.knex.raw(`SET LOCAL ROLE ??`, [roleName]).toQuery());
}
- private async resetRole(prisma: { $executeRawUnsafe(query: string): Promise }) {
- await prisma.$executeRawUnsafe(this.knex.raw(`RESET ROLE`).toQuery());
+ private async setTransactionReadOnly(prisma: {
+ $executeRawUnsafe(query: string): Promise;
+ }) {
+ await prisma.$executeRawUnsafe('SET TRANSACTION READ ONLY');
}
- private async readonlyExecuteSql(baseId: string, sql: string) {
- return this.databaseRouter.dataPrismaTransactionForBase(baseId, async (prisma) => {
- await prisma.$executeRawUnsafe('SET TRANSACTION READ ONLY');
- return await prisma.$queryRawUnsafe(sql);
- });
+ private async setLocalStatementTimeout(prisma: {
+ $executeRawUnsafe(query: string): Promise;
+ }) {
+ const timeoutMs = this.thresholdConfig.searchTimeout;
+ await prisma.$executeRawUnsafe(
+ this.knex.raw(`SET LOCAL statement_timeout = ?`, [timeoutMs]).toQuery()
+ );
}
/**
@@ -197,23 +208,7 @@ export class BaseSqlExecutorService {
database: this.driver,
});
// 3. read only role check table access, only pg and pg version > 14 support
- try {
- await this.readonlyExecuteSql(baseId, sql);
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- } catch (error: any) {
- throw new CustomHttpException(
- `read only check failed: ${error?.meta?.message || error?.message}`,
- HttpErrorCode.VALIDATION_ERROR,
- {
- localization: {
- i18nKey: 'httpErrors.baseSqlExecutor.readOnlyCheckFailed',
- context: {
- message: error?.meta?.message || error?.message,
- },
- },
- }
- );
- }
+ // TODO: need read only db connection for better security
}
async executeQuerySql(
@@ -225,10 +220,14 @@ export class BaseSqlExecutorService {
}
) {
await this.safeCheckSql(baseId, sql, opts);
- await this.roleCheckAndCreate(baseId);
+ const shouldSetLocalRole = await this.roleCheckAndCreate(baseId);
return this.databaseRouter.dataPrismaTransactionForBase(baseId, async (prisma) => {
try {
- await this.setRole(prisma, baseId);
+ await this.setLocalStatementTimeout(prisma);
+ await this.setTransactionReadOnly(prisma);
+ if (shouldSetLocalRole) {
+ await this.setLocalRole(prisma, baseId);
+ }
return await prisma.$queryRawUnsafe(sql);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} catch (error: any) {
@@ -244,10 +243,6 @@ export class BaseSqlExecutorService {
},
}
);
- } finally {
- await this.resetRole(prisma).catch((error) => {
- console.log('resetRole error', error);
- });
}
});
}
diff --git a/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts b/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts
index dc37594fc4..e618634237 100644
--- a/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts
+++ b/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts
@@ -13,6 +13,14 @@ describe('base sql executor utils', () => {
expect(() => validateRoleOperations('set role xxx;')).toThrow();
});
+ it('should throw an error if the sql contains set local role', () => {
+ expect(() => validateRoleOperations('set local role xxx')).toThrow();
+ });
+
+ it('should throw an error if the sql contains set session role', () => {
+ expect(() => validateRoleOperations('set session role xxx')).toThrow();
+ });
+
it('should throw an error if the sql contains set role with line break', () => {
expect(() =>
validateRoleOperations(`set
@@ -136,5 +144,74 @@ describe('base sql executor utils', () => {
})
).not.toThrow();
});
+
+ it.each([
+ 'SELECT count(*) FROM "bseXXX"."tblYYY"',
+ 'SELECT sum("amount") FROM "bseXXX"."tblYYY"',
+ 'SELECT abs("amount") FROM "bseXXX"."tblYYY"',
+ 'SELECT round("amount", 2) FROM "bseXXX"."tblYYY"',
+ 'SELECT floor("amount") FROM "bseXXX"."tblYYY"',
+ 'SELECT ceil("amount") FROM "bseXXX"."tblYYY"',
+ 'SELECT ceiling("amount") FROM "bseXXX"."tblYYY"',
+ 'SELECT lower("name") FROM "bseXXX"."tblYYY"',
+ 'SELECT replace("name", \'a\', \'b\') FROM "bseXXX"."tblYYY"',
+ 'SELECT regexp_replace("name", \'a\', \'b\') FROM "bseXXX"."tblYYY"',
+ 'SELECT substring("name" from 1 for 3) FROM "bseXXX"."tblYYY"',
+ 'SELECT substr("name", 1, 3) FROM "bseXXX"."tblYYY"',
+ 'SELECT concat("first_name", "last_name") FROM "bseXXX"."tblYYY"',
+ 'SELECT concat_ws(\' \', "first_name", "last_name") FROM "bseXXX"."tblYYY"',
+ 'SELECT split_part("name", \'/\', 1) FROM "bseXXX"."tblYYY"',
+ 'SELECT coalesce("name", \'unknown\') FROM "bseXXX"."tblYYY"',
+ 'SELECT to_char("created_time", \'YYYY-MM-DD\') FROM "bseXXX"."tblYYY"',
+ 'SELECT extract(year from "created_time") FROM "bseXXX"."tblYYY"',
+ 'SELECT json_extract_path_text("payload"::json, \'name\') FROM "bseXXX"."tblYYY"',
+ ])('allows explicitly permitted function in %s', (sql) => {
+ expect(() =>
+ checkTableAccess(sql, {
+ tableNames: ['bseXXX.tblYYY'],
+ database: DriverClient.Pg,
+ })
+ ).not.toThrow();
+ });
+
+ it.each([
+ [
+ "SELECT query_to_xml('SELECT rolname FROM pg_catalog.pg_roles', true, false, '')",
+ 'query_to_xml',
+ ],
+ [
+ "SELECT query_to_xmlschema('SELECT relname FROM pg_catalog.pg_class', true, false, '')",
+ 'query_to_xmlschema',
+ ],
+ [
+ "SELECT query_to_xml_and_xmlschema('SELECT table_name FROM information_schema.tables', true, false, '')",
+ 'query_to_xml_and_xmlschema',
+ ],
+ ["SELECT ts_stat('SELECT to_tsvector(''simple'', ''abc'')')", 'ts_stat'],
+ ["SELECT set_config('statement_timeout','0',true)", 'set_config'],
+ ['SELECT lo_create(0)', 'lo_create'],
+ ["SELECT lo_from_bytea(0, decode('414243','hex'))", 'lo_from_bytea'],
+ ['SELECT pg_try_advisory_lock(123456789)', 'pg_try_advisory_lock'],
+ ['SELECT pg_advisory_unlock_all()', 'pg_advisory_unlock_all'],
+ [
+ "WITH n AS (SELECT pg_notify('cuppy_probe', 'blackbox')) SELECT 'ok' AS result FROM n",
+ 'pg_notify',
+ ],
+ ["SELECT pg_logical_emit_message(false, 'cuppy_probe', 'hello')", 'pg_logical_emit_message'],
+ ['SELECT pg_export_snapshot()', 'pg_export_snapshot'],
+ ['SELECT pg_lock_status()', 'pg_lock_status'],
+ ['SELECT pg_control_system()', 'pg_control_system'],
+ ['SELECT pg_current_wal_lsn()', 'pg_current_wal_lsn'],
+ ['SELECT pg_database_size(current_database())', 'pg_database_size'],
+ ['SELECT version()', 'version'],
+ ["SELECT pg_catalog.pg_notify('cuppy_probe', 'blackbox')", 'pg_notify'],
+ ])('blocks unapproved function %s', (sql, functionName) => {
+ expect(() =>
+ checkTableAccess(sql, {
+ tableNames: [],
+ database: DriverClient.Pg,
+ })
+ ).toThrow(new RegExp(`function ${functionName}`));
+ });
});
});
diff --git a/apps/nestjs-backend/src/features/base-sql-executor/utils.ts b/apps/nestjs-backend/src/features/base-sql-executor/utils.ts
index a56e1922e1..0474f82dc9 100644
--- a/apps/nestjs-backend/src/features/base-sql-executor/utils.ts
+++ b/apps/nestjs-backend/src/features/base-sql-executor/utils.ts
@@ -2,6 +2,9 @@ import { DriverClient, HttpErrorCode } from '@teable/core';
import type { AST } from 'node-sql-parser';
import { Parser } from 'node-sql-parser';
import { CustomHttpException } from '../../custom.exception';
+import { allowedFunctions } from './allowed-functions';
+
+const whiteListCheckErrorKey = 'httpErrors.baseSqlExecutor.whiteListCheckError';
export const validateRoleOperations = (sql: string) => {
const removeQuotedContent = (sql: string) => {
@@ -11,7 +14,11 @@ export const validateRoleOperations = (sql: string) => {
const normalizedSql = sql.toLowerCase().replace(/\s+/g, ' ');
const sqlWithoutQuotes = removeQuotedContent(normalizedSql);
- const roleOperationPatterns = [/set\s+role/, /reset\s+role/, /set\s+session/];
+ const roleOperationPatterns = [
+ /set\s+(?:local\s+|session\s+)?role/,
+ /reset\s+role/,
+ /set\s+session/,
+ ];
for (const pattern of roleOperationPatterns) {
if (pattern.test(sqlWithoutQuotes)) {
@@ -35,6 +42,79 @@ const databaseTypeMap = {
[DriverClient.Pg]: 'postgresql',
};
+const getFunctionName = (node: unknown): string | null => {
+ const functionNode = node as {
+ type?: unknown;
+ name?: { name?: Array<{ value?: unknown }> };
+ };
+ if (functionNode.type !== 'function') {
+ return null;
+ }
+
+ const nameParts = functionNode.name?.name;
+ const lastNamePart = nameParts?.[nameParts.length - 1]?.value;
+ return typeof lastNamePart === 'string' ? lastNamePart.toLowerCase() : null;
+};
+
+const findUnallowedFunctionInArray = (values: unknown[]): string | null => {
+ for (const value of values) {
+ const unallowed = findUnallowedFunction(value);
+ if (unallowed) {
+ return unallowed;
+ }
+ }
+
+ return null;
+};
+
+const findUnallowedFunctionInValue = (value: unknown): string | null => {
+ if (Array.isArray(value)) {
+ return findUnallowedFunctionInArray(value);
+ }
+
+ return findUnallowedFunction(value);
+};
+
+function findUnallowedFunction(node: unknown): string | null {
+ if (!node || typeof node !== 'object') {
+ return null;
+ }
+
+ const functionName = getFunctionName(node);
+ if (functionName && !allowedFunctions.has(functionName)) {
+ return functionName;
+ }
+
+ for (const value of Object.values(node)) {
+ const unallowed = findUnallowedFunctionInValue(value);
+ if (unallowed) {
+ return unallowed;
+ }
+ }
+
+ return null;
+}
+
+const validateFunctionCalls = (ast: AST | AST[]) => {
+ const unallowed = findUnallowedFunction(ast);
+ if (!unallowed) {
+ return;
+ }
+
+ throw new CustomHttpException(
+ `not allowed to execute sql with function ${unallowed}`,
+ HttpErrorCode.VALIDATION_ERROR,
+ {
+ localization: {
+ i18nKey: whiteListCheckErrorKey,
+ context: {
+ function: unallowed,
+ },
+ },
+ }
+ );
+};
+
const collectWithNames = (ast?: AST) => {
if (!ast) {
return [];
@@ -79,6 +159,7 @@ export const checkTableAccess = (
);
}
})();
+ validateFunctionCalls(ast);
const withNames = Array.isArray(ast) ? ast.flatMap(collectWithNames) : collectWithNames(ast);
const allowedTables = new Set([...withNames, ...tableNames]);
const whiteColumnList = Array.from(allowedTables).map((table) => {
@@ -95,6 +176,18 @@ export const checkTableAccess = (
}
const sqlTableList = parser.tableList(sql, opt);
+
+ if (!sqlTableList.length) {
+ throw new CustomHttpException(
+ 'SQL syntax error or no table accessed, please check your query',
+ HttpErrorCode.VALIDATION_ERROR,
+ {
+ localization: {
+ i18nKey: 'httpErrors.baseSqlExecutor.sqlSyntaxError',
+ },
+ }
+ );
+ }
const invalidTableNames = sqlTableList
.filter((t: string) => !whiteColumnList.includes(t))
.map((t: string) => t.split('::').pop()!);
@@ -111,7 +204,7 @@ export const checkTableAccess = (
HttpErrorCode.VALIDATION_ERROR,
{
localization: {
- i18nKey: 'httpErrors.baseSqlExecutor.whiteListCheckError',
+ i18nKey: whiteListCheckErrorKey,
context: {
message,
},
diff --git a/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts b/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts
index fc0f545ebb..746ad4dcaf 100644
--- a/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts
+++ b/apps/nestjs-backend/src/features/base/base-duplicate.service.spec.ts
@@ -2,11 +2,10 @@ import type { TestingModule } from '@nestjs/testing';
import { Test } from '@nestjs/testing';
import { FieldType, Relationship } from '@teable/core';
import { BaseDuplicateMode } from '@teable/openapi';
-import { v2RecordRepositoryPostgresTokens } from '@teable/v2-adapter-table-repository-postgres';
-import { TableByIdSpec, v2CoreTokens } from '@teable/v2-core';
+import { v2CoreTokens, type DuplicateBaseRecordReadOptions } from '@teable/v2-core';
import { GlobalModule } from '../../global/global.module';
import { BaseDuplicateService } from './base-duplicate.service';
-import type { BaseImportProgressCallback, IBaseImportProgress } from './base-import.service';
+import type { IBaseImportProgress } from './base-import.service';
import { BaseModule } from './base.module';
import type { ILinkFieldTableMap } from './utils';
@@ -150,7 +149,6 @@ describe('BaseDuplicateService normalizeDuplicateStructureForV2', () => {
});
describe('BaseDuplicateService duplicateBaseV2', () => {
- const okResult = (value: T) => ({ isErr: () => false, value });
type IServiceArgs = ConstructorParameters;
const duplicateBaseName = 'Duplicated base';
const sourceTableName = 'Source table';
@@ -160,15 +158,20 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
buildDuplicateStructureConfig: (...args: unknown[]) => Promise;
getCrossBaseLinkFieldTableMap: (...args: unknown[]) => Promise;
getDisconnectedLinkFieldTableMap: (...args: unknown[]) => Promise;
+ getV2CrossBaseLinkFieldTableMap: (...args: unknown[]) => Promise;
+ getV2DisconnectedLinkFieldTableMap: (...args: unknown[]) => Promise;
+ getV2InternalLinkRelationTableMap: (...args: unknown[]) => Promise>;
getDisconnectedLinkFieldIds: (...args: unknown[]) => Promise;
normalizeDuplicateStructureForV2: (structure: unknown) => unknown;
createDuplicateBaseSource: (...args: unknown[]) => {
- records(tableId: string): AsyncIterable<{ fields: Record }>;
+ records(
+ tableId: string,
+ options?: DuplicateBaseRecordReadOptions
+ ): AsyncIterable<{ fields: Record }>;
};
duplicateTableData: (...args: unknown[]) => Promise;
duplicateAttachments: (...args: unknown[]) => Promise;
duplicateLinkJunction: (...args: unknown[]) => Promise;
- backfillDuplicatedBaseComputedFields: (...args: unknown[]) => Promise;
};
it('should create the v2 execution context from the space data container', async () => {
@@ -231,9 +234,9 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
{} as IServiceArgs[6],
{ get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7],
{} as IServiceArgs[8],
- {} as IServiceArgs[9],
- v2ContainerService as unknown as IServiceArgs[10],
- v2ContextFactory as unknown as IServiceArgs[11]
+ v2ContainerService as unknown as IServiceArgs[9],
+ v2ContextFactory as unknown as IServiceArgs[10],
+ {} as IServiceArgs[11]
);
const internals = service as unknown as IDuplicateServiceInternals;
@@ -243,9 +246,9 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
structure,
sourceDbTableNameByTableId,
});
- vi.spyOn(internals, 'getCrossBaseLinkFieldTableMap').mockResolvedValue({});
- vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue({});
- vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue([]);
+ vi.spyOn(internals, 'getV2CrossBaseLinkFieldTableMap').mockResolvedValue({});
+ vi.spyOn(internals, 'getV2DisconnectedLinkFieldTableMap').mockResolvedValue({});
+ vi.spyOn(internals, 'getV2InternalLinkRelationTableMap').mockResolvedValue({});
vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure);
vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source);
@@ -262,15 +265,17 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
'bseSource',
structure,
{},
- sourceDbTableNameByTableId
+ sourceDbTableNameByTableId,
+ {}
);
expect(commandBus.execute).toHaveBeenCalledWith(context, expect.any(Object));
});
- it('should create v2 structure first and copy records with raw table duplication', async () => {
+ it('should copy direct duplicate records through bulk SQL after v2 structure creation', async () => {
const spaceId = 'spcTarget';
const targetBaseId = 'bseTarget';
const tableIdMap = { tblSource: 'tblTarget' };
+ const targetTableId = tableIdMap.tblSource;
const fieldIdMap = { fldLink: 'fldTargetLink' };
const viewIdMap = { viwSource: 'viwTarget' };
const context = { requestId: 'ctx' };
@@ -335,10 +340,10 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
{} as IServiceArgs[5],
persistedComputedBackfillService as unknown as IServiceArgs[6],
{ get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7],
- {} as IServiceArgs[8],
- dataDbClientManager as unknown as IServiceArgs[9],
- v2ContainerService as unknown as IServiceArgs[10],
- v2ContextFactory as unknown as IServiceArgs[11]
+ dataDbClientManager as unknown as IServiceArgs[8],
+ v2ContainerService as unknown as IServiceArgs[9],
+ v2ContextFactory as unknown as IServiceArgs[10],
+ {} as IServiceArgs[11]
);
const internals = service as unknown as IDuplicateServiceInternals;
const mergedLinkFieldTableMap = {
@@ -349,17 +354,17 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
structure,
sourceDbTableNameByTableId: { tblSource: sourceDbTableName },
});
- vi.spyOn(internals, 'getCrossBaseLinkFieldTableMap').mockResolvedValue({});
- vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue(
+ vi.spyOn(internals, 'getV2CrossBaseLinkFieldTableMap').mockResolvedValue({});
+ vi.spyOn(internals, 'getV2DisconnectedLinkFieldTableMap').mockResolvedValue(
mergedLinkFieldTableMap
);
+ vi.spyOn(internals, 'getV2InternalLinkRelationTableMap').mockResolvedValue({});
vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue(['fldDisconnected']);
vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure);
vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source);
vi.spyOn(internals, 'duplicateTableData').mockResolvedValue(12);
vi.spyOn(internals, 'duplicateAttachments').mockResolvedValue(undefined);
vi.spyOn(internals, 'duplicateLinkJunction').mockResolvedValue(undefined);
- vi.spyOn(internals, 'backfillDuplicatedBaseComputedFields').mockResolvedValue(undefined);
const result = await service.duplicateBaseV2({
fromBaseId: 'bseSource',
@@ -368,15 +373,18 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
withRecords: true,
});
- const executedCommand = commandBus.execute.mock.calls[0]?.[1] as { withRecords: boolean };
+ const executedCommand = commandBus.execute.mock.calls[0]?.[1] as {
+ withRecords: boolean;
+ batchSize: number;
+ };
expect(executedCommand.withRecords).toBe(false);
+ expect(executedCommand.batchSize).toBe(500);
expect(internals.duplicateTableData).toHaveBeenCalledWith(
targetBaseId,
tableIdMap,
fieldIdMap,
viewIdMap,
- mergedLinkFieldTableMap,
- undefined
+ mergedLinkFieldTableMap
);
expect(internals.duplicateLinkJunction).toHaveBeenCalledWith(
targetBaseId,
@@ -385,69 +393,130 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
true,
['fldDisconnected']
);
- expect(persistedComputedBackfillService.recomputeForTables).toHaveBeenCalledWith(['tblTarget']);
- expect(internals.backfillDuplicatedBaseComputedFields).toHaveBeenCalledWith(
- container,
- context,
- ['tblTarget']
+ expect(persistedComputedBackfillService.recomputeForTables).toHaveBeenCalledWith([
+ targetTableId,
+ ]);
+ expect(internals.duplicateAttachments).toHaveBeenCalledWith(
+ targetBaseId,
+ tableIdMap,
+ fieldIdMap
);
expect(result.recordsLength).toBe(12);
});
- it('should synchronously backfill v2 computed and link fields after raw record copy', async () => {
- const targetTableId = 'tblaaaaaaaaaaaaaaaa';
+ it('should stream v2 duplicate records when source and target data databases differ', async () => {
+ const spaceId = 'spcTarget';
+ const targetBaseId = 'bseTarget';
+ const tableIdMap = { tblSource: 'tblTarget' };
+ const fieldIdMap = { fldText: 'fldTargetText' };
+ const viewIdMap = { viwSource: 'viwTarget' };
const context = { requestId: 'ctx' };
- const fields = [{ name: 'Owner' }];
- const table = {
- getFields: vi.fn().mockReturnValue(fields),
+ const db = { dialect: 'pg' };
+ const structure = {
+ name: duplicateBaseName,
+ icon: undefined,
+ tables: [{ id: 'tblSource', name: sourceTableName, fields: [], views: [] }],
};
- const tableRepository = {
- findOne: vi.fn().mockResolvedValue(okResult(table)),
+ const source = {
+ structure,
+ records: async function* () {
+ yield undefined as never;
+ },
};
- const backfillService = {
- executeSyncMany: vi.fn().mockResolvedValue(okResult(undefined)),
+ const commandBus = {
+ execute: vi.fn().mockResolvedValue({
+ isErr: () => false,
+ value: (async function* () {
+ yield {
+ id: 'done',
+ baseId: targetBaseId,
+ tableIdMap,
+ fieldIdMap,
+ viewIdMap,
+ recordsLength: 7,
+ };
+ })(),
+ }),
};
const container = {
- resolve: vi.fn((token: symbol) => {
- if (token === v2CoreTokens.tableRepository) {
- return tableRepository;
- }
- if (token === v2RecordRepositoryPostgresTokens.computedFieldBackfillService) {
- return backfillService;
- }
- throw new Error('Unexpected token');
- }),
+ resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(db),
+ };
+ const baseUpdate = vi.fn();
+ const baseImportService = {
+ createBaseV2: vi.fn().mockResolvedValue({ id: targetBaseId }),
+ restoreBaseExtrasV2: vi.fn().mockResolvedValue({ appIdMap: {}, workflowIdMap: {} }),
+ };
+ const dataDbClientManager = {
+ getDataDatabaseForBase: vi
+ .fn()
+ .mockResolvedValueOnce({ cacheKey: 'source-db' })
+ .mockResolvedValueOnce({ cacheKey: 'target-db' }),
+ };
+ const v2ContainerService = {
+ getContainerForSpace: vi.fn().mockResolvedValue(container),
+ };
+ const v2ContextFactory = {
+ createContext: vi.fn().mockResolvedValue(context),
};
const service = new BaseDuplicateService(
- {} as IServiceArgs[0],
+ {
+ txClient: vi.fn().mockReturnValue({
+ base: { update: baseUpdate },
+ }),
+ } as unknown as IServiceArgs[0],
{} as IServiceArgs[1],
{} as IServiceArgs[2],
- {} as IServiceArgs[3],
+ baseImportService as unknown as IServiceArgs[3],
{} as IServiceArgs[4],
{} as IServiceArgs[5],
{} as IServiceArgs[6],
- {} as IServiceArgs[7],
- {} as IServiceArgs[8],
- {} as IServiceArgs[9],
- {} as IServiceArgs[10],
+ { get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7],
+ dataDbClientManager as unknown as IServiceArgs[8],
+ v2ContainerService as unknown as IServiceArgs[9],
+ v2ContextFactory as unknown as IServiceArgs[10],
{} as IServiceArgs[11]
);
const internals = service as unknown as IDuplicateServiceInternals;
- await internals.backfillDuplicatedBaseComputedFields(container, context, [targetTableId]);
+ vi.spyOn(internals, 'buildDuplicateStructureConfig').mockResolvedValue({
+ structure,
+ sourceDbTableNameByTableId: { tblSource: sourceDbTableName },
+ });
+ vi.spyOn(internals, 'getV2CrossBaseLinkFieldTableMap').mockResolvedValue({});
+ vi.spyOn(internals, 'getV2DisconnectedLinkFieldTableMap').mockResolvedValue({});
+ vi.spyOn(internals, 'getV2InternalLinkRelationTableMap').mockResolvedValue({});
+ vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue([]);
+ vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure);
+ vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source);
+ vi.spyOn(internals, 'duplicateTableData').mockResolvedValue(0);
+ vi.spyOn(internals, 'duplicateAttachments').mockResolvedValue(undefined);
+ vi.spyOn(internals, 'duplicateLinkJunction').mockResolvedValue(undefined);
- expect(container.resolve).toHaveBeenCalledWith(v2CoreTokens.tableRepository);
- expect(container.resolve).toHaveBeenCalledWith(
- v2RecordRepositoryPostgresTokens.computedFieldBackfillService
- );
- expect(tableRepository.findOne).toHaveBeenCalledWith(context, expect.any(TableByIdSpec));
- expect(backfillService.executeSyncMany).toHaveBeenCalledWith(context, {
- table,
- fields,
- skipDistinctFilter: true,
- includeOneManyTwoWay: true,
+ const result = await service.duplicateBaseV2({
+ fromBaseId: 'bseSource',
+ spaceId,
+ name: duplicateBaseName,
+ withRecords: true,
+ });
+
+ expect(dataDbClientManager.getDataDatabaseForBase).toHaveBeenCalledWith('bseSource', {
+ useTransaction: true,
});
+ expect(dataDbClientManager.getDataDatabaseForBase).toHaveBeenCalledWith(targetBaseId, {
+ useTransaction: true,
+ });
+ expect((commandBus.execute.mock.calls[0]?.[1] as { withRecords: boolean }).withRecords).toBe(
+ true
+ );
+ expect(internals.duplicateTableData).not.toHaveBeenCalled();
+ expect(internals.duplicateLinkJunction).not.toHaveBeenCalled();
+ expect(internals.duplicateAttachments).toHaveBeenCalledWith(
+ targetBaseId,
+ tableIdMap,
+ fieldIdMap
+ );
+ expect(result.recordsLength).toBe(7);
});
it('should forward real row totals through duplicateBaseV2 progress events', async () => {
@@ -473,13 +542,35 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
execute: vi.fn().mockResolvedValue({
isErr: () => false,
value: (async function* () {
+ yield {
+ id: 'progress',
+ phase: 'table_data_start',
+ processedRows: 0,
+ totalRows: 12,
+ };
+ yield {
+ id: 'progress',
+ phase: 'table_data_progress',
+ tableId: 'tblTarget',
+ tableName: sourceTableName,
+ processedRows: 5,
+ batchProcessedRows: 5,
+ currentBatch: 1,
+ totalRows: 12,
+ };
+ yield {
+ id: 'progress',
+ phase: 'table_data_done',
+ processedRows: 12,
+ totalRows: 12,
+ };
yield {
id: 'done',
baseId: targetBaseId,
tableIdMap,
fieldIdMap,
viewIdMap,
- recordsLength: 0,
+ recordsLength: 12,
};
})(),
}),
@@ -517,10 +608,10 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
{} as IServiceArgs[5],
persistedComputedBackfillService as unknown as IServiceArgs[6],
{ get: vi.fn().mockReturnValue('usrTest') } as unknown as IServiceArgs[7],
- {} as IServiceArgs[8],
- dataDbClientManager as unknown as IServiceArgs[9],
- v2ContainerService as unknown as IServiceArgs[10],
- v2ContextFactory as unknown as IServiceArgs[11]
+ dataDbClientManager as unknown as IServiceArgs[8],
+ v2ContainerService as unknown as IServiceArgs[9],
+ v2ContextFactory as unknown as IServiceArgs[10],
+ {} as IServiceArgs[11]
);
const internals = service as unknown as IDuplicateServiceInternals;
const progressEvents: unknown[] = [];
@@ -529,29 +620,14 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
structure,
sourceDbTableNameByTableId: { tblSource: sourceDbTableName },
});
- vi.spyOn(internals, 'getCrossBaseLinkFieldTableMap').mockResolvedValue({});
- vi.spyOn(internals, 'getDisconnectedLinkFieldTableMap').mockResolvedValue({});
- vi.spyOn(internals, 'getDisconnectedLinkFieldIds').mockResolvedValue([]);
+ vi.spyOn(internals, 'getV2CrossBaseLinkFieldTableMap').mockResolvedValue({});
+ vi.spyOn(internals, 'getV2DisconnectedLinkFieldTableMap').mockResolvedValue({});
+ vi.spyOn(internals, 'getV2InternalLinkRelationTableMap').mockResolvedValue({});
vi.spyOn(internals, 'normalizeDuplicateStructureForV2').mockReturnValue(structure);
vi.spyOn(internals, 'createDuplicateBaseSource').mockReturnValue(source);
- vi.spyOn(internals, 'duplicateTableData').mockImplementation(async (...args: unknown[]) => {
- const onProgress = args[5] as BaseImportProgressCallback | undefined;
- onProgress?.({ phase: 'table_data_start', processedRows: 0, totalRows: 12 });
- onProgress?.({
- phase: 'table_data_progress',
- tableId: 'tblTarget',
- tableName: sourceTableName,
- processedRows: 5,
- batchProcessedRows: 5,
- currentBatch: 1,
- totalRows: 12,
- });
- onProgress?.({ phase: 'table_data_done', processedRows: 12, totalRows: 12 });
- return 12;
- });
+ vi.spyOn(internals, 'duplicateTableData').mockResolvedValue(12);
vi.spyOn(internals, 'duplicateAttachments').mockResolvedValue(undefined);
vi.spyOn(internals, 'duplicateLinkJunction').mockResolvedValue(undefined);
- vi.spyOn(internals, 'backfillDuplicatedBaseComputedFields').mockResolvedValue(undefined);
const result = await service.duplicateBaseV2(
{
@@ -565,13 +641,9 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
(event: string | IBaseImportProgress) => progressEvents.push(event)
);
- expect(internals.duplicateTableData).toHaveBeenCalledWith(
- targetBaseId,
- tableIdMap,
- fieldIdMap,
- viewIdMap,
- {},
- expect.any(Function)
+ expect(internals.duplicateTableData).not.toHaveBeenCalled();
+ expect((commandBus.execute.mock.calls[0]?.[1] as { withRecords: boolean }).withRecords).toBe(
+ true
);
expect(progressEvents).toEqual(
expect.arrayContaining([
@@ -661,10 +733,10 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
knex as unknown as IServiceArgs[5],
{} as IServiceArgs[6],
{} as IServiceArgs[7],
- {} as IServiceArgs[8],
{
dataPrismaForBase: vi.fn().mockResolvedValue(dataPrisma),
- } as unknown as IServiceArgs[9],
+ } as unknown as IServiceArgs[8],
+ {} as IServiceArgs[9],
{} as IServiceArgs[10],
{} as IServiceArgs[11]
);
@@ -724,10 +796,10 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
{} as IServiceArgs[5],
{} as IServiceArgs[6],
{} as IServiceArgs[7],
- {} as IServiceArgs[8],
{
dataKnexForBase: vi.fn().mockResolvedValue(dataKnex),
- } as unknown as IServiceArgs[9],
+ } as unknown as IServiceArgs[8],
+ {} as IServiceArgs[9],
{} as IServiceArgs[10],
{} as IServiceArgs[11]
);
@@ -768,7 +840,503 @@ describe('BaseDuplicateService duplicateBaseV2', () => {
recordId: 'recSource',
fields: { fldText: 'A' },
autoNumber: 1,
+ lastModifiedTime: null,
+ lastModifiedBy: null,
},
]);
});
+
+ it('should skip internal v2 link relation reads during insert phase', async () => {
+ const sourceTableQuery = {
+ select: vi.fn().mockReturnThis(),
+ where: vi.fn().mockReturnThis(),
+ orderBy: vi.fn().mockReturnThis(),
+ limit: vi
+ .fn()
+ .mockResolvedValueOnce([
+ {
+ __id: 'recSource',
+ __auto_number: 1,
+ fldStory: [{ id: 'recStale', title: 'Deleted story' }],
+ },
+ ])
+ .mockResolvedValueOnce([]),
+ };
+ const dataKnex = vi.fn((tableName: string) => {
+ if (tableName === 'bseSource.tblSource') return sourceTableQuery;
+ throw new Error(`unexpected table ${tableName}`);
+ });
+ const service = new BaseDuplicateService(
+ {} as IServiceArgs[0],
+ {} as IServiceArgs[1],
+ {} as IServiceArgs[2],
+ {} as IServiceArgs[3],
+ {} as IServiceArgs[4],
+ {} as IServiceArgs[5],
+ {} as IServiceArgs[6],
+ {} as IServiceArgs[7],
+ { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex) } as unknown as IServiceArgs[8],
+ {} as IServiceArgs[9],
+ {} as IServiceArgs[10],
+ {} as IServiceArgs[11]
+ );
+ const internals = service as unknown as IDuplicateServiceInternals;
+ const source = internals.createDuplicateBaseSource(
+ 'bseSource',
+ {
+ name: duplicateBaseName,
+ tables: [
+ {
+ id: 'tblSource',
+ name: sourceTableName,
+ dbTableName: 'tblShortName',
+ fields: [
+ {
+ id: 'fldStory',
+ name: 'Story',
+ dbFieldName: 'fldStory',
+ type: FieldType.Link,
+ },
+ ],
+ views: [],
+ },
+ ],
+ },
+ {},
+ { tblSource: 'bseSource.tblSource' },
+ {
+ tblSource: [
+ {
+ fieldId: 'fldStory',
+ dbFieldName: 'fldStory',
+ foreignTableId: 'tblStory',
+ lookupFieldId: 'fldStoryName',
+ relationship: 'manyMany',
+ fkHostTableName: 'bseSource.junction_fldStory',
+ selfKeyName: '__fk_self',
+ foreignKeyName: '__fk_foreign',
+ isOneWay: false,
+ isMultipleCellValue: true,
+ orderColumnName: '__order',
+ },
+ ],
+ }
+ );
+
+ const records = [];
+ for await (const record of source.records('tblSource', { phase: 'insert' })) {
+ records.push(record);
+ }
+
+ expect(dataKnex).toHaveBeenCalledWith('bseSource.tblSource');
+ expect(dataKnex).not.toHaveBeenCalledWith('bseSource.junction_fldStory');
+ expect(records[0].fields.fldStory).toEqual([{ id: 'recStale', title: 'Deleted story' }]);
+ });
+
+ it('should rebuild internal v2 link values from relation storage and ignore stale cache ids', async () => {
+ const sourceTableQuery = {
+ select: vi.fn().mockReturnThis(),
+ where: vi.fn().mockReturnThis(),
+ orderBy: vi.fn().mockReturnThis(),
+ limit: vi
+ .fn()
+ .mockResolvedValueOnce([
+ {
+ __id: 'recSource',
+ __auto_number: 1,
+ fldStory: [
+ { id: 'recExisting', title: 'Existing story' },
+ { id: 'recStale', title: 'Deleted story' },
+ ],
+ },
+ ])
+ .mockResolvedValueOnce([]),
+ };
+ const junctionQuery = {
+ select: vi.fn().mockReturnThis(),
+ whereIn: vi.fn().mockReturnThis(),
+ orderBy: vi.fn().mockResolvedValue([
+ {
+ sourceRecordId: 'recSource',
+ foreignRecordId: 'recExisting',
+ },
+ ]),
+ };
+ const dataKnex = vi.fn((tableName: string) => {
+ if (tableName === 'bseSource.tblSource') return sourceTableQuery;
+ if (tableName === 'bseSource.junction_fldStory') return junctionQuery;
+ throw new Error(`unexpected table ${tableName}`);
+ });
+ const service = new BaseDuplicateService(
+ {} as IServiceArgs[0],
+ {} as IServiceArgs[1],
+ {} as IServiceArgs[2],
+ {} as IServiceArgs[3],
+ {} as IServiceArgs[4],
+ {} as IServiceArgs[5],
+ {} as IServiceArgs[6],
+ {} as IServiceArgs[7],
+ { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex) } as unknown as IServiceArgs[8],
+ {} as IServiceArgs[9],
+ {} as IServiceArgs[10],
+ {} as IServiceArgs[11]
+ );
+ const internals = service as unknown as IDuplicateServiceInternals;
+ const source = internals.createDuplicateBaseSource(
+ 'bseSource',
+ {
+ name: duplicateBaseName,
+ tables: [
+ {
+ id: 'tblSource',
+ name: sourceTableName,
+ dbTableName: 'tblShortName',
+ fields: [
+ {
+ id: 'fldStory',
+ name: 'Story',
+ dbFieldName: 'fldStory',
+ type: FieldType.Link,
+ },
+ ],
+ views: [],
+ },
+ ],
+ },
+ {},
+ { tblSource: 'bseSource.tblSource' },
+ {
+ tblSource: [
+ {
+ fieldId: 'fldStory',
+ dbFieldName: 'fldStory',
+ foreignTableId: 'tblStory',
+ lookupFieldId: 'fldStoryName',
+ relationship: 'manyMany',
+ fkHostTableName: 'bseSource.junction_fldStory',
+ selfKeyName: '__fk_self',
+ foreignKeyName: '__fk_foreign',
+ isOneWay: false,
+ isMultipleCellValue: true,
+ orderColumnName: '__order',
+ },
+ ],
+ }
+ );
+
+ const records = [];
+ for await (const record of source.records('tblSource', { phase: 'linkRestore' })) {
+ records.push(record);
+ }
+
+ expect(dataKnex).toHaveBeenCalledWith('bseSource.tblSource');
+ expect(dataKnex).toHaveBeenCalledWith('bseSource.junction_fldStory');
+ expect(junctionQuery.whereIn).toHaveBeenCalledWith('__fk_self', ['recSource']);
+ expect(records[0].fields.fldStory).toEqual([{ id: 'recExisting' }]);
+ });
+
+ it('should rebuild many-one internal v2 link values from current table FK storage', async () => {
+ const sourceTableQuery = {
+ select: vi.fn().mockReturnThis(),
+ where: vi.fn().mockReturnThis(),
+ orderBy: vi.fn().mockReturnThis(),
+ limit: vi
+ .fn()
+ .mockResolvedValueOnce([
+ {
+ __id: 'recChild',
+ __auto_number: 1,
+ fldParent: { id: 'recStale', title: 'Deleted parent' },
+ },
+ ])
+ .mockResolvedValueOnce([]),
+ whereIn: vi.fn().mockReturnThis(),
+ whereNotNull: vi.fn().mockResolvedValue([
+ {
+ sourceRecordId: 'recChild',
+ foreignRecordId: 'recParent',
+ },
+ ]),
+ };
+ const dataKnex = vi.fn((tableName: string) => {
+ if (tableName === 'bseSource.childTable') return sourceTableQuery;
+ throw new Error(`unexpected table ${tableName}`);
+ });
+ const service = new BaseDuplicateService(
+ {} as IServiceArgs[0],
+ {} as IServiceArgs[1],
+ {} as IServiceArgs[2],
+ {} as IServiceArgs[3],
+ {} as IServiceArgs[4],
+ {} as IServiceArgs[5],
+ {} as IServiceArgs[6],
+ {} as IServiceArgs[7],
+ { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex) } as unknown as IServiceArgs[8],
+ {} as IServiceArgs[9],
+ {} as IServiceArgs[10],
+ {} as IServiceArgs[11]
+ );
+ const internals = service as unknown as IDuplicateServiceInternals;
+ const source = internals.createDuplicateBaseSource(
+ 'bseSource',
+ {
+ name: duplicateBaseName,
+ tables: [
+ {
+ id: 'tblChild',
+ name: sourceTableName,
+ dbTableName: 'childTable',
+ fields: [
+ {
+ id: 'fldParent',
+ name: 'Parent',
+ dbFieldName: 'fldParent',
+ type: FieldType.Link,
+ },
+ ],
+ views: [],
+ },
+ ],
+ },
+ {},
+ { tblChild: 'bseSource.childTable' },
+ {
+ tblChild: [
+ {
+ fieldId: 'fldParent',
+ dbFieldName: 'fldParent',
+ foreignTableId: 'tblParent',
+ lookupFieldId: 'fldParentName',
+ relationship: 'manyOne',
+ fkHostTableName: 'bseSource.childTable',
+ selfKeyName: '__id',
+ foreignKeyName: '__fk_parent',
+ isOneWay: false,
+ isMultipleCellValue: false,
+ orderColumnName: '__fk_parent_order',
+ },
+ ],
+ }
+ );
+
+ const records = [];
+ for await (const record of source.records('tblChild', { phase: 'linkRestore' })) {
+ records.push(record);
+ }
+
+ expect(dataKnex).toHaveBeenCalledWith('bseSource.childTable');
+ expect(sourceTableQuery.whereIn).toHaveBeenCalledWith('__id', ['recChild']);
+ expect(sourceTableQuery.whereNotNull).toHaveBeenCalledWith('__fk_parent');
+ expect(records[0].fields.fldParent).toEqual({ id: 'recParent' });
+ });
+
+ it('should rebuild two-way one-many internal v2 link values from foreign table FK storage', async () => {
+ const sourceTableQuery = {
+ select: vi.fn().mockReturnThis(),
+ where: vi.fn().mockReturnThis(),
+ orderBy: vi.fn().mockReturnThis(),
+ limit: vi
+ .fn()
+ .mockResolvedValueOnce([
+ {
+ __id: 'recParent',
+ __auto_number: 1,
+ fldChildren: [
+ { id: 'recChildExisting', title: 'Existing child' },
+ { id: 'recChildStale', title: 'Deleted child' },
+ ],
+ },
+ ])
+ .mockResolvedValueOnce([]),
+ };
+ const foreignTableQuery = {
+ select: vi.fn().mockReturnThis(),
+ whereIn: vi.fn().mockReturnThis(),
+ orderBy: vi.fn().mockResolvedValue([
+ {
+ sourceRecordId: 'recParent',
+ foreignRecordId: 'recChildExisting',
+ },
+ ]),
+ };
+ const dataKnex = vi.fn((tableName: string) => {
+ if (tableName === 'bseSource.parentTable') return sourceTableQuery;
+ if (tableName === 'bseSource.childTable') return foreignTableQuery;
+ throw new Error(`unexpected table ${tableName}`);
+ });
+ const service = new BaseDuplicateService(
+ {} as IServiceArgs[0],
+ {} as IServiceArgs[1],
+ {} as IServiceArgs[2],
+ {} as IServiceArgs[3],
+ {} as IServiceArgs[4],
+ {} as IServiceArgs[5],
+ {} as IServiceArgs[6],
+ {} as IServiceArgs[7],
+ { dataKnexForBase: vi.fn().mockResolvedValue(dataKnex) } as unknown as IServiceArgs[8],
+ {} as IServiceArgs[9],
+ {} as IServiceArgs[10],
+ {} as IServiceArgs[11]
+ );
+ const internals = service as unknown as IDuplicateServiceInternals;
+ const source = internals.createDuplicateBaseSource(
+ 'bseSource',
+ {
+ name: duplicateBaseName,
+ tables: [
+ {
+ id: 'tblParent',
+ name: sourceTableName,
+ dbTableName: 'parentTable',
+ fields: [
+ {
+ id: 'fldChildren',
+ name: 'Children',
+ dbFieldName: 'fldChildren',
+ type: FieldType.Link,
+ },
+ ],
+ views: [],
+ },
+ ],
+ },
+ {},
+ { tblParent: 'bseSource.parentTable' },
+ {
+ tblParent: [
+ {
+ fieldId: 'fldChildren',
+ dbFieldName: 'fldChildren',
+ foreignTableId: 'tblChild',
+ lookupFieldId: 'fldChildName',
+ relationship: 'oneMany',
+ fkHostTableName: 'bseSource.childTable',
+ selfKeyName: '__fk_parent',
+ foreignKeyName: '__id',
+ isOneWay: false,
+ isMultipleCellValue: true,
+ orderColumnName: '__fk_parent_order',
+ },
+ ],
+ }
+ );
+
+ const records = [];
+ for await (const record of source.records('tblParent', { phase: 'linkRestore' })) {
+ records.push(record);
+ }
+
+ expect(dataKnex).toHaveBeenCalledWith('bseSource.childTable');
+ expect(foreignTableQuery.whereIn).toHaveBeenCalledWith('__fk_parent', ['recParent']);
+ expect(foreignTableQuery.orderBy).toHaveBeenCalledWith('__fk_parent_order', 'asc');
+ expect(records[0].fields.fldChildren).toEqual([{ id: 'recChildExisting' }]);
+ });
+
+ it('should normalize postgres array literal link values when downgrading v2 cross-base links', async () => {
+ const query = {
+ select: vi.fn().mockReturnThis(),
+ where: vi.fn().mockReturnThis(),
+ orderBy: vi.fn().mockReturnThis(),
+ limit: vi
+ .fn()
+ .mockResolvedValueOnce([
+ {
+ __id: 'recSource',
+ __auto_number: 1,
+ fldVendor: '{"{\\"id\\":\\"recVendor\\",\\"title\\":\\"Vendor A\\"}"}',
+ },
+ ])
+ .mockResolvedValueOnce([]),
+ };
+ const dataKnex = vi.fn().mockReturnValue(query);
+ const service = new BaseDuplicateService(
+ {} as IServiceArgs[0],
+ {} as IServiceArgs[1],
+ {} as IServiceArgs[2],
+ {} as IServiceArgs[3],
+ {} as IServiceArgs[4],
+ {} as IServiceArgs[5],
+ {} as IServiceArgs[6],
+ {} as IServiceArgs[7],
+ {
+ dataKnexForBase: vi.fn().mockResolvedValue(dataKnex),
+ } as unknown as IServiceArgs[8],
+ {} as IServiceArgs[9],
+ {} as IServiceArgs[10],
+ {} as IServiceArgs[11]
+ );
+ const internals = service as unknown as IDuplicateServiceInternals;
+ const source = internals.createDuplicateBaseSource(
+ 'bseSource',
+ {
+ name: duplicateBaseName,
+ tables: [
+ {
+ id: 'tblSource',
+ name: sourceTableName,
+ dbTableName: 'tblSource',
+ fields: [
+ {
+ id: 'fldVendor',
+ name: 'Vendor',
+ dbFieldName: 'fldVendor',
+ type: FieldType.SingleLineText,
+ },
+ ],
+ views: [],
+ },
+ ],
+ },
+ {
+ tblSource: [
+ {
+ dbFieldName: 'fldVendor',
+ selfKeyName: 'fk_fld_vendor',
+ isMultipleCellValue: true,
+ },
+ ],
+ },
+ { tblSource: 'bseSource.tblSource' }
+ );
+
+ const records = [];
+ for await (const record of source.records('tblSource')) {
+ records.push(record);
+ }
+
+ expect(records[0].fields.fldVendor).toBe('Vendor A');
+ });
+
+ it('should include nullable isLookup host link fields when building v2 cross-base maps', async () => {
+ const fieldFindMany = vi.fn().mockResolvedValue([]);
+ const service = new BaseDuplicateService(
+ {
+ txClient: vi.fn().mockReturnValue({
+ field: { findMany: fieldFindMany },
+ }),
+ } as unknown as IServiceArgs[0],
+ {} as IServiceArgs[1],
+ {} as IServiceArgs[2],
+ {} as IServiceArgs[3],
+ {} as IServiceArgs[4],
+ {} as IServiceArgs[5],
+ {} as IServiceArgs[6],
+ {} as IServiceArgs[7],
+ {} as IServiceArgs[8],
+ {} as IServiceArgs[9],
+ {} as IServiceArgs[10],
+ {} as IServiceArgs[11]
+ );
+ const internals = service as unknown as IDuplicateServiceInternals;
+
+ await internals.getV2CrossBaseLinkFieldTableMap({ tblSource: 'tblSource' });
+
+ expect(fieldFindMany).toHaveBeenCalledWith(
+ expect.objectContaining({
+ where: expect.objectContaining({
+ OR: [{ isLookup: false }, { isLookup: null }],
+ }),
+ })
+ );
+ });
});
diff --git a/apps/nestjs-backend/src/features/base/base-duplicate.service.ts b/apps/nestjs-backend/src/features/base/base-duplicate.service.ts
index 8a5f0202a8..92fb40eb0d 100644
--- a/apps/nestjs-backend/src/features/base/base-duplicate.service.ts
+++ b/apps/nestjs-backend/src/features/base/base-duplicate.service.ts
@@ -10,26 +10,18 @@ import {
type IDuplicateBaseRo,
} from '@teable/openapi';
import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg';
-import {
- v2RecordRepositoryPostgresTokens,
- type ComputedFieldBackfillService,
-} from '@teable/v2-adapter-table-repository-postgres';
import {
DuplicateBaseCommand,
- TableByIdSpec,
- TableId,
v2CoreTokens,
+ type DuplicateBaseRecordReadOptions,
type DotTeaFieldInput,
type DuplicateBaseResult,
type DuplicateBaseSource,
type ICommandBus,
- type IExecutionContext,
- type ITableRepository,
type NormalizedDotTeaField,
type NormalizedDotTeaStructure,
} from '@teable/v2-core';
-import type { DependencyContainer } from '@teable/v2-di';
-import { normalizeField } from '@teable/v2-dottea';
+import { normalizeFields } from '@teable/v2-dottea';
import { Knex } from 'knex';
import type { Kysely } from 'kysely';
import { groupBy, omit } from 'lodash';
@@ -58,8 +50,10 @@ import { mergeLinkFieldTableMaps } from './utils';
import type { ILinkFieldTableInfo, ILinkFieldTableMap } from './utils';
type DuplicatedBase = Awaited>['base'];
-const v2DuplicateReadBatchSize = 500;
+const v2DuplicateReadBatchSize = 1000;
const v2DuplicateCopyBatchSize = 500;
+const v2DuplicateLinkFieldBatchSize = 500;
+const v2DuplicateTableIdQueryChunkSize = 100;
type DuplicateStructureConfig = Awaited>;
type DuplicateV2FieldConfig = Omit<
DuplicateStructureConfig['tables'][number]['fields'][number],
@@ -77,6 +71,33 @@ type DuplicateStructureConfigResult = {
structure: DuplicateStructureConfig;
sourceDbTableNameByTableId: Record;
};
+type DuplicateBaseStructConfigInput = Parameters<
+ BaseExportService['generateBaseStructConfig']
+>[0] & {
+ includeWorkflowRuntimeState?: boolean;
+};
+type DuplicateLinkFieldRaw = {
+ id: string;
+ tableId: string;
+ dbFieldName: string;
+ isMultipleCellValue: boolean | null;
+ meta: string | null;
+ options: string | null;
+};
+type V2InternalLinkFieldTableInfo = {
+ fieldId: string;
+ dbFieldName: string;
+ foreignTableId: string;
+ lookupFieldId: string;
+ relationship: string;
+ fkHostTableName: string;
+ selfKeyName: string;
+ foreignKeyName: string;
+ isOneWay: boolean;
+ isMultipleCellValue: boolean;
+ orderColumnName?: string;
+};
+type V2LinkCellItem = { id: string; title?: string };
type IDataPrismaExecutor = {
$executeRawUnsafe(query: string, ...values: unknown[]): Promise;
@@ -269,9 +290,9 @@ export class BaseDuplicateService {
// allowCrossBase=true keeps same-space cross-base links intact. Mirrors
// the v1 branch in duplicateBase above.
const crossBaseLinkFieldTableMap: ILinkFieldTableMap = allowCrossBase
- ? await this.getCrossBaseLinkFieldTableMap(sourceTableIdMap, spaceId)
- : await this.getCrossBaseLinkFieldTableMap(sourceTableIdMap);
- const disconnectedLinkFieldTableMap = await this.getDisconnectedLinkFieldTableMap(
+ ? await this.getV2CrossBaseLinkFieldTableMap(sourceTableIdMap, spaceId)
+ : await this.getV2CrossBaseLinkFieldTableMap(sourceTableIdMap);
+ const disconnectedLinkFieldTableMap = await this.getV2DisconnectedLinkFieldTableMap(
sourceTableIdMap,
fromBaseId,
nodes,
@@ -281,12 +302,8 @@ export class BaseDuplicateService {
crossBaseLinkFieldTableMap,
disconnectedLinkFieldTableMap
);
- const disconnectedLinkFieldIds = await this.getDisconnectedLinkFieldIds(
- sourceTableIdMap,
- fromBaseId,
- nodes,
- skipParentNodes
- );
+ const internalLinkRelationTableMap =
+ await this.getV2InternalLinkRelationTableMap(sourceTableIdMap);
const container = await this.v2ContainerService.getContainerForSpace(spaceId);
const commandBus = container.resolve(v2CoreTokens.commandBus);
const db = container.resolve>(v2PostgresDbTokens.db);
@@ -299,8 +316,11 @@ export class BaseDuplicateService {
baseId,
duplicateMode !== BaseDuplicateMode.CopyShareBase
);
+ const useBulkRecordCopy =
+ Boolean(withRecords) &&
+ !onProgress &&
+ (await this.isSameDataDatabaseForRecordCopy(fromBaseId, base.id));
if (withRecords) {
- await this.assertSameDataDatabaseForRecordCopy(fromBaseId, base.id);
await prisma.base.update({
where: { id: base.id },
data: {
@@ -315,12 +335,14 @@ export class BaseDuplicateService {
fromBaseId,
normalizedStructure,
mergedLinkFieldTableMap,
- sourceDbTableNameByTableId
+ sourceDbTableNameByTableId,
+ internalLinkRelationTableMap
);
const commandResult = DuplicateBaseCommand.createFromSource({
baseId: base.id,
source,
- withRecords: false,
+ withRecords: Boolean(withRecords) && !useBulkRecordCopy,
+ batchSize: v2DuplicateCopyBatchSize,
});
if (commandResult.isErr()) {
throw new Error(commandResult.error.message);
@@ -360,37 +382,38 @@ export class BaseDuplicateService {
structure,
{ tableIdMap, fieldIdMap, viewIdMap },
duplicateMode,
- onProgress,
- { restoreEeResources: true }
+ onProgress
);
if (withRecords) {
- recordsLength = await this.duplicateTableData(
- base.id,
- tableIdMap,
- fieldIdMap,
- viewIdMap,
- mergedLinkFieldTableMap,
- onProgress
- );
+ if (useBulkRecordCopy) {
+ recordsLength = await this.duplicateTableData(
+ base.id,
+ tableIdMap,
+ fieldIdMap,
+ viewIdMap,
+ mergedLinkFieldTableMap
+ );
+ const disconnectedLinkFieldIds = await this.getDisconnectedLinkFieldIds(
+ tableIdMap,
+ fromBaseId,
+ nodes,
+ skipParentNodes
+ );
+ await this.duplicateLinkJunction(
+ base.id,
+ tableIdMap,
+ fieldIdMap,
+ allowCrossBase,
+ disconnectedLinkFieldIds
+ );
+ await this.persistedComputedBackfillService.recomputeForTables(Object.values(tableIdMap));
+ }
onProgress?.({
phase: 'attachments_copying',
processedRows: recordsLength,
totalRows: recordsLength,
});
await this.duplicateAttachments(base.id, tableIdMap, fieldIdMap);
- await this.duplicateLinkJunction(
- base.id,
- tableIdMap,
- fieldIdMap,
- allowCrossBase,
- disconnectedLinkFieldIds
- );
- await this.persistedComputedBackfillService.recomputeForTables(Object.values(tableIdMap));
- await this.backfillDuplicatedBaseComputedFields(
- container,
- context,
- Object.values(tableIdMap)
- );
await prisma.base.update({
where: { id: base.id },
data: {
@@ -454,12 +477,7 @@ export class BaseDuplicateService {
}
private async assertSameDataDatabaseForRecordCopy(sourceBaseId: string, targetBaseId: string) {
- const [source, target] = await Promise.all([
- this.dataDbClientManager.getDataDatabaseForBase(sourceBaseId, { useTransaction: true }),
- this.dataDbClientManager.getDataDatabaseForBase(targetBaseId, { useTransaction: true }),
- ]);
-
- if (source.cacheKey === target.cacheKey) {
+ if (await this.isSameDataDatabaseForRecordCopy(sourceBaseId, targetBaseId)) {
return;
}
@@ -469,6 +487,15 @@ export class BaseDuplicateService {
);
}
+ private async isSameDataDatabaseForRecordCopy(sourceBaseId: string, targetBaseId: string) {
+ const [source, target] = await Promise.all([
+ this.dataDbClientManager.getDataDatabaseForBase(sourceBaseId, { useTransaction: true }),
+ this.dataDbClientManager.getDataDatabaseForBase(targetBaseId, { useTransaction: true }),
+ ]);
+
+ return source.cacheKey === target.cacheKey;
+ }
+
private async buildDuplicateStructureConfig(
fromBaseId: string,
baseName?: string,
@@ -509,23 +536,10 @@ export class BaseDuplicateService {
},
});
const tableIds = tableRaws.map(({ id }) => id);
- const fieldRaws = await prisma.field.findMany({
- where: {
- tableId: { in: tableIds },
- deletedTime: null,
- },
- });
- const viewRaws = await prisma.view.findMany({
- where: {
- tableId: { in: tableIds },
- deletedTime: null,
- },
- orderBy: {
- order: 'asc',
- },
- });
+ const fieldRaws = await this.baseExportService.findFieldsByTableIds(tableIds);
+ const viewRaws = await this.baseExportService.findViewsByTableIds(tableIds);
- const structure = await this.baseExportService.generateBaseStructConfig({
+ const structureInput: DuplicateBaseStructConfigInput = {
baseRaw,
tableRaws,
fieldRaws,
@@ -539,7 +553,9 @@ export class BaseDuplicateService {
excludedTableIds,
rootNodeIds,
destSpaceId,
- });
+ includeWorkflowRuntimeState: false,
+ };
+ const structure = await this.baseExportService.generateBaseStructConfig(structureInput);
return {
structure,
@@ -555,26 +571,38 @@ export class BaseDuplicateService {
sourceBaseId: string,
structure: DuplicateV2StructureConfig,
disconnectedLinkFieldTableMap: ILinkFieldTableMap,
- sourceDbTableNameByTableId: Record = {}
+ sourceDbTableNameByTableId: Record = {},
+ internalLinkRelationTableMap: Record = {}
): DuplicateBaseSource {
const tableById = new Map(structure.tables.map((table) => [table.id, table]));
- const readRows = (sourceDbTableName: string, crossBaseLinkInfo: ILinkFieldTableInfo[]) =>
- this.createSourceTableRecordRows(sourceBaseId, sourceDbTableName, crossBaseLinkInfo);
+ const readRows = (
+ sourceDbTableName: string,
+ crossBaseLinkInfo: ILinkFieldTableInfo[],
+ internalLinkInfo: V2InternalLinkFieldTableInfo[]
+ ) =>
+ this.createSourceTableRecordRows(
+ sourceBaseId,
+ sourceDbTableName,
+ crossBaseLinkInfo,
+ internalLinkInfo
+ );
const toRecordInput = (table: DuplicateV2TableConfig, row: Record) =>
this.toDuplicateBaseRecordInput(table, row);
return {
structure,
- async *records(tableId: string) {
+ async *records(tableId: string, options?: DuplicateBaseRecordReadOptions) {
const table = tableById.get(tableId);
const sourceDbTableName = sourceDbTableNameByTableId[tableId] ?? table?.dbTableName;
if (!table || !sourceDbTableName) {
return;
}
+ const shouldReadInternalLinks = options?.phase !== 'insert';
for await (const row of readRows(
sourceDbTableName,
- disconnectedLinkFieldTableMap[tableId] || []
+ disconnectedLinkFieldTableMap[tableId] || [],
+ shouldReadInternalLinks ? internalLinkRelationTableMap[tableId] || [] : []
)) {
yield toRecordInput(table, row);
}
@@ -616,29 +644,13 @@ export class BaseDuplicateService {
return {
...structure,
tables: structure.tables.map((table) => {
- const tableFieldTypesById = new Map(
- table.fields
- .filter((field) => field.id)
- .map(
- (field) =>
- [
- field.id!,
- field.isConditionalLookup
- ? 'conditionalLookup'
- : field.isLookup
- ? 'lookup'
- : field.type,
- ] as const
- )
- );
-
return {
...table,
- fields: table.fields.map((field) => {
- const normalized = normalizeField(field as DotTeaFieldInput, tableFieldTypesById, {
- availableTableIds,
- fieldIdsByTableId,
- });
+ fields: normalizeFields(table.fields as ReadonlyArray, {
+ availableTableIds,
+ fieldIdsByTableId,
+ }).map((normalized, index) => {
+ const field = table.fields[index]!;
const normalizedField = { ...field, ...normalized };
if (
@@ -723,12 +735,8 @@ export class BaseDuplicateService {
...(row.__auto_number ? { autoNumber: Number(row.__auto_number) } : {}),
...(row.__created_time ? { createdTime: this.toRestoreString(row.__created_time) } : {}),
...(row.__created_by ? { createdBy: this.toRestoreString(row.__created_by) } : {}),
- ...(row.__last_modified_time
- ? { lastModifiedTime: this.toRestoreString(row.__last_modified_time) }
- : {}),
- ...(row.__last_modified_by
- ? { lastModifiedBy: this.toRestoreString(row.__last_modified_by) }
- : {}),
+ lastModifiedTime: null,
+ lastModifiedBy: null,
};
}
@@ -1013,6 +1021,211 @@ export class BaseDuplicateService {
return tableId2DbFieldNameMap;
}
+ private async findV2DuplicateLinkFields(tableIds: string[]) {
+ const prisma = this.prismaService.txClient();
+ const fields: DuplicateLinkFieldRaw[] = [];
+
+ for (let index = 0; index < tableIds.length; index += v2DuplicateTableIdQueryChunkSize) {
+ const tableIdChunk = tableIds.slice(index, index + v2DuplicateTableIdQueryChunkSize);
+ let cursor: string | undefined;
+ let hasMore = true;
+
+ while (hasMore) {
+ const page = await prisma.field.findMany({
+ where: {
+ tableId: { in: tableIdChunk },
+ type: FieldType.Link,
+ OR: [{ isLookup: false }, { isLookup: null }],
+ deletedTime: null,
+ },
+ ...(cursor
+ ? {
+ cursor: {
+ id: cursor,
+ },
+ skip: 1,
+ }
+ : {}),
+ select: {
+ id: true,
+ tableId: true,
+ dbFieldName: true,
+ isMultipleCellValue: true,
+ meta: true,
+ options: true,
+ },
+ orderBy: [{ tableId: 'asc' }, { id: 'asc' }],
+ take: v2DuplicateLinkFieldBatchSize,
+ });
+
+ fields.push(...page);
+
+ hasMore = page.length === v2DuplicateLinkFieldBatchSize;
+ if (hasMore) {
+ cursor = page[page.length - 1].id;
+ }
+ }
+ }
+
+ return fields;
+ }
+
+ private parseLinkFieldOptions(options: string | null): ILinkFieldOptions | undefined {
+ if (!options) {
+ return undefined;
+ }
+
+ try {
+ return JSON.parse(options) as ILinkFieldOptions;
+ } catch {
+ return undefined;
+ }
+ }
+
+ private resolveV2LinkOrderColumnName(options: ILinkFieldOptions): string | undefined {
+ switch (options.relationship) {
+ case 'manyMany':
+ return '__order';
+ case 'oneMany':
+ return options.isOneWay ? '__order' : `${options.selfKeyName}_order`;
+ case 'manyOne':
+ case 'oneOne':
+ return `${options.foreignKeyName}_order`;
+ default:
+ return undefined;
+ }
+ }
+
+ private hasV2LinkOrderColumn(meta: string | null): boolean {
+ if (!meta) {
+ return false;
+ }
+
+ try {
+ return Boolean((JSON.parse(meta) as { hasOrderColumn?: unknown }).hasOrderColumn);
+ } catch {
+ return false;
+ }
+ }
+
+ private buildLinkFieldTableMap(
+ tableIdMap: Record,
+ fields: DuplicateLinkFieldRaw[]
+ ): ILinkFieldTableMap {
+ const tableId2DbFieldNameMap: ILinkFieldTableMap = {};
+
+ Object.entries(groupBy(fields, 'tableId')).forEach(([tableId, fields]) => {
+ const info = fields.flatMap(({ dbFieldName, isMultipleCellValue, options }) => {
+ const parsedOptions = this.parseLinkFieldOptions(options);
+ if (!parsedOptions?.selfKeyName) {
+ return [];
+ }
+ return [
+ {
+ dbFieldName,
+ selfKeyName: parsedOptions.selfKeyName,
+ isMultipleCellValue: !!isMultipleCellValue,
+ },
+ ];
+ });
+
+ if (info.length) {
+ tableId2DbFieldNameMap[tableId] = info;
+ tableId2DbFieldNameMap[tableIdMap[tableId]] = info;
+ }
+ });
+
+ return tableId2DbFieldNameMap;
+ }
+
+ private async getV2InternalLinkRelationTableMap(
+ tableIdMap: Record
+ ): Promise> {
+ const tableIds = Object.keys(tableIdMap);
+ const fields = (await this.findV2DuplicateLinkFields(tableIds)).filter((field) => {
+ const options = this.parseLinkFieldOptions(field.options);
+ return (
+ options?.foreignTableId &&
+ !options.baseId &&
+ tableIdMap[options.foreignTableId] &&
+ options.fkHostTableName &&
+ options.selfKeyName &&
+ options.foreignKeyName
+ );
+ });
+
+ if (!fields.length) {
+ return {};
+ }
+
+ const tableId2Info: Record = {};
+ Object.entries(groupBy(fields, 'tableId')).forEach(([tableId, tableFields]) => {
+ const info = tableFields.flatMap((field) => {
+ const options = this.parseLinkFieldOptions(field.options);
+ const foreignTableId = options?.foreignTableId;
+ const lookupFieldId = options?.lookupFieldId;
+ if (
+ !foreignTableId ||
+ !lookupFieldId ||
+ !options?.relationship ||
+ !options.fkHostTableName ||
+ !options.selfKeyName ||
+ !options.foreignKeyName
+ ) {
+ return [];
+ }
+
+ return [
+ {
+ fieldId: field.id,
+ dbFieldName: field.dbFieldName,
+ foreignTableId,
+ lookupFieldId,
+ relationship: options.relationship,
+ fkHostTableName: options.fkHostTableName,
+ selfKeyName: options.selfKeyName,
+ foreignKeyName: options.foreignKeyName,
+ isOneWay: Boolean(options.isOneWay),
+ isMultipleCellValue: !!field.isMultipleCellValue,
+ orderColumnName: this.hasV2LinkOrderColumn(field.meta)
+ ? this.resolveV2LinkOrderColumnName(options)
+ : undefined,
+ },
+ ];
+ });
+
+ if (info.length) {
+ tableId2Info[tableId] = info;
+ tableId2Info[tableIdMap[tableId]] = info;
+ }
+ });
+
+ return tableId2Info;
+ }
+
+ private async getV2DisconnectedLinkFieldTableMap(
+ tableIdMap: Record,
+ fromBaseId: string,
+ nodes?: string[],
+ skipParentNodes: boolean = false
+ ): Promise {
+ const { excludedTableIds } = await this.collectNodesAndResourceIds(
+ fromBaseId,
+ nodes,
+ skipParentNodes
+ );
+
+ if (!nodes?.length || !excludedTableIds?.length) {
+ return {};
+ }
+
+ const fields = (await this.findV2DuplicateLinkFields(Object.keys(tableIdMap))).filter((field) =>
+ excludedTableIds.includes(this.parseLinkFieldOptions(field.options)?.foreignTableId ?? '')
+ );
+
+ return this.buildLinkFieldTableMap(tableIdMap, fields);
+ }
+
async previewCrossSpaceAffectedFields(
fromBaseId: string,
destSpaceId: string
@@ -1127,6 +1340,42 @@ export class BaseDuplicateService {
return tableId2DbFieldNameMap;
}
+ private async getV2CrossBaseLinkFieldTableMap(
+ tableIdMap: Record,
+ destSpaceId?: string
+ ): Promise {
+ const linkFields = (await this.findV2DuplicateLinkFields(Object.keys(tableIdMap))).filter(
+ (field) => this.parseLinkFieldOptions(field.options)?.baseId
+ );
+
+ let crossBaseLinkFields = linkFields;
+ if (destSpaceId) {
+ const prisma = this.prismaService.txClient();
+ const foreignBaseIds = Array.from(
+ new Set(
+ linkFields
+ .map((field) => this.parseLinkFieldOptions(field.options)?.baseId)
+ .filter((baseId): baseId is string => Boolean(baseId))
+ )
+ );
+ const bases = foreignBaseIds.length
+ ? await prisma.base.findMany({
+ where: { id: { in: foreignBaseIds }, deletedTime: null },
+ select: { id: true, spaceId: true },
+ })
+ : [];
+ const crossSpaceBaseIds = new Set(
+ bases.filter((base) => base.spaceId !== destSpaceId).map((base) => base.id)
+ );
+ crossBaseLinkFields = linkFields.filter((field) => {
+ const baseId = this.parseLinkFieldOptions(field.options)?.baseId;
+ return baseId && crossSpaceBaseIds.has(baseId);
+ });
+ }
+
+ return this.buildLinkFieldTableMap(tableIdMap, crossBaseLinkFields);
+ }
+
private async duplicateTableData(
targetBaseId: string,
tableIdMap: Record,
@@ -1396,10 +1645,64 @@ export class BaseDuplicateService {
return String(value);
}
+ private parsePostgresTextArrayLiteral(value: string): string[] | undefined {
+ if (!value.startsWith('{') || !value.endsWith('}')) {
+ return undefined;
+ }
+
+ const inner = value.slice(1, -1);
+ if (!inner) {
+ return [];
+ }
+
+ const items: string[] = [];
+ let item = '';
+ let inQuotes = false;
+
+ for (let index = 0; index < inner.length; index += 1) {
+ const char = inner[index];
+
+ if (inQuotes) {
+ if (char === '\\') {
+ index += 1;
+ item += inner[index] ?? '';
+ continue;
+ }
+ if (char === '"') {
+ inQuotes = false;
+ continue;
+ }
+ item += char;
+ continue;
+ }
+
+ if (char === '"') {
+ inQuotes = true;
+ continue;
+ }
+
+ if (char === ',') {
+ items.push(item);
+ item = '';
+ continue;
+ }
+
+ item += char;
+ }
+
+ if (inQuotes) {
+ return undefined;
+ }
+
+ items.push(item);
+ return items;
+ }
+
private async *createSourceTableRecordRows(
sourceBaseId: string,
sourceDbTableName: string,
- crossBaseLinkInfo: ILinkFieldTableInfo[]
+ crossBaseLinkInfo: ILinkFieldTableInfo[],
+ internalLinkInfo: V2InternalLinkFieldTableInfo[] = []
): AsyncGenerator> {
const dataKnex = await this.dataDbClientManager.dataKnexForBase(sourceBaseId, {
useTransaction: true,
@@ -1416,8 +1719,18 @@ export class BaseDuplicateService {
return;
}
- for (const row of rows) {
- yield this.normalizeCrossBaseLinkColumns(row, crossBaseLinkInfo);
+ const sourceRecordIds = rows.flatMap((row) =>
+ typeof row.__id === 'string' ? [row.__id] : []
+ );
+ const normalizedRows = await this.normalizeInternalLinkColumns(
+ dataKnex,
+ rows.map((row) => this.normalizeCrossBaseLinkColumns(row, crossBaseLinkInfo)),
+ sourceRecordIds,
+ internalLinkInfo
+ );
+
+ for (const row of normalizedRows) {
+ yield row;
}
lastAutoNumber = Number(rows[rows.length - 1]?.__auto_number ?? lastAutoNumber);
@@ -1444,6 +1757,138 @@ export class BaseDuplicateService {
return nextRow;
}
+ private async normalizeInternalLinkColumns(
+ dataKnex: Knex,
+ rows: Record[],
+ sourceRecordIds: string[],
+ internalLinkInfo: V2InternalLinkFieldTableInfo[]
+ ) {
+ if (!rows.length || !internalLinkInfo.length) {
+ return rows;
+ }
+
+ const rowsById = new Map(
+ rows.flatMap((row) => (typeof row.__id === 'string' ? ([[row.__id, row]] as const) : []))
+ );
+ for (const row of rows) {
+ for (const { dbFieldName } of internalLinkInfo) {
+ delete row[dbFieldName];
+ }
+ }
+
+ if (!sourceRecordIds.length) {
+ return rows;
+ }
+
+ for (const info of internalLinkInfo) {
+ const relations = await this.findV2InternalLinkRelations(dataKnex, sourceRecordIds, info);
+ for (const [sourceRecordId, linkItems] of relations) {
+ const row = rowsById.get(sourceRecordId);
+ if (!row) continue;
+ if (info.isMultipleCellValue) {
+ row[info.dbFieldName] = linkItems;
+ } else if (linkItems[0]) {
+ row[info.dbFieldName] = linkItems[0];
+ }
+ }
+ }
+
+ return rows;
+ }
+
+ private async findV2InternalLinkRelations(
+ dataKnex: Knex,
+ sourceRecordIds: string[],
+ info: V2InternalLinkFieldTableInfo
+ ): Promise> {
+ if (info.relationship === 'manyMany' || (info.relationship === 'oneMany' && info.isOneWay)) {
+ return this.findV2JunctionLinkRelations(dataKnex, sourceRecordIds, info);
+ }
+
+ if (
+ (info.relationship === 'manyOne' || info.relationship === 'oneOne') &&
+ info.foreignKeyName !== '__id'
+ ) {
+ return this.findV2CurrentTableFkLinkRelations(dataKnex, sourceRecordIds, info);
+ }
+
+ if (
+ info.relationship === 'oneMany' ||
+ ((info.relationship === 'manyOne' || info.relationship === 'oneOne') &&
+ info.foreignKeyName === '__id')
+ ) {
+ return this.findV2ForeignTableFkLinkRelations(dataKnex, sourceRecordIds, info);
+ }
+
+ return new Map();
+ }
+
+ private async findV2JunctionLinkRelations(
+ dataKnex: Knex,
+ sourceRecordIds: string[],
+ info: V2InternalLinkFieldTableInfo
+ ): Promise> {
+ const relationRows = await dataKnex(info.fkHostTableName)
+ .select({
+ sourceRecordId: info.selfKeyName,
+ foreignRecordId: info.foreignKeyName,
+ })
+ .whereIn(info.selfKeyName, sourceRecordIds)
+ .orderBy(info.orderColumnName || info.foreignKeyName, 'asc');
+
+ return this.groupLinkRelationRows(relationRows);
+ }
+
+ private async findV2CurrentTableFkLinkRelations(
+ dataKnex: Knex,
+ sourceRecordIds: string[],
+ info: V2InternalLinkFieldTableInfo
+ ): Promise> {
+ const relationRows = await dataKnex(info.fkHostTableName)
+ .select({
+ sourceRecordId: info.selfKeyName,
+ foreignRecordId: info.foreignKeyName,
+ })
+ .whereIn(info.selfKeyName, sourceRecordIds)
+ .whereNotNull(info.foreignKeyName);
+
+ return this.groupLinkRelationRows(relationRows);
+ }
+
+ private async findV2ForeignTableFkLinkRelations(
+ dataKnex: Knex,
+ sourceRecordIds: string[],
+ info: V2InternalLinkFieldTableInfo
+ ): Promise> {
+ const query = dataKnex(info.fkHostTableName)
+ .select({
+ sourceRecordId: info.selfKeyName,
+ foreignRecordId: info.foreignKeyName,
+ })
+ .whereIn(info.selfKeyName, sourceRecordIds);
+ const relationRows = info.orderColumnName
+ ? await query.orderBy(info.orderColumnName, 'asc')
+ : await query;
+
+ return this.groupLinkRelationRows(relationRows);
+ }
+
+ private groupLinkRelationRows(
+ relationRows: Array<{ sourceRecordId?: unknown; foreignRecordId?: unknown }>
+ ): Map {
+ const relations = new Map();
+ for (const { sourceRecordId, foreignRecordId } of relationRows) {
+ if (typeof sourceRecordId !== 'string' || typeof foreignRecordId !== 'string') {
+ continue;
+ }
+ const items = relations.get(sourceRecordId) ?? [];
+ items.push({ id: foreignRecordId });
+ relations.set(sourceRecordId, items);
+ }
+
+ return relations;
+ }
+
private toCrossBaseLinkTitle(value: unknown, isMultipleCellValue: boolean): unknown {
if (value == null) {
return value;
@@ -1453,21 +1898,19 @@ export class BaseDuplicateService {
try {
return this.toCrossBaseLinkTitle(JSON.parse(value), isMultipleCellValue);
} catch {
+ const arrayValue = this.parsePostgresTextArrayLiteral(value);
+ if (arrayValue) {
+ return this.toCrossBaseLinkTitle(arrayValue, isMultipleCellValue);
+ }
return value;
}
}
- if (isMultipleCellValue) {
- return Array.isArray(value)
- ? value
- .map((item) =>
- item && typeof item === 'object' && 'title' in item
- ? String((item as { title?: unknown }).title ?? '')
- : ''
- )
- .filter(Boolean)
- .join(', ')
- : value;
+ if (Array.isArray(value)) {
+ const titles = value
+ .map((item) => this.toCrossBaseLinkTitle(item, false))
+ .filter((item): item is string => typeof item === 'string' && Boolean(item));
+ return isMultipleCellValue ? titles.join(', ') : titles[0];
}
return value && typeof value === 'object' && 'title' in value
@@ -1515,44 +1958,4 @@ export class BaseDuplicateService {
disconnectedLinkFieldIds
);
}
-
- private async backfillDuplicatedBaseComputedFields(
- container: DependencyContainer,
- context: IExecutionContext,
- targetTableIds: string[]
- ) {
- if (!targetTableIds.length) {
- return;
- }
-
- const tableRepository = container.resolve(v2CoreTokens.tableRepository);
- const backfillService = container.resolve(
- v2RecordRepositoryPostgresTokens.computedFieldBackfillService
- );
-
- for (const rawTableId of targetTableIds) {
- const tableIdResult = TableId.create(rawTableId);
- if (tableIdResult.isErr()) {
- throw new Error(tableIdResult.error.message);
- }
-
- const tableResult = await tableRepository.findOne(
- context,
- TableByIdSpec.create(tableIdResult.value)
- );
- if (tableResult.isErr()) {
- throw new Error(tableResult.error.message);
- }
-
- const backfillResult = await backfillService.executeSyncMany(context, {
- table: tableResult.value,
- fields: tableResult.value.getFields(),
- skipDistinctFilter: true,
- includeOneManyTwoWay: true,
- });
- if (backfillResult.isErr()) {
- throw new Error(backfillResult.error.message);
- }
- }
- }
}
diff --git a/apps/nestjs-backend/src/features/base/base-export.service.ts b/apps/nestjs-backend/src/features/base/base-export.service.ts
index 850368fdfc..e61dc63d0e 100644
--- a/apps/nestjs-backend/src/features/base/base-export.service.ts
+++ b/apps/nestjs-backend/src/features/base/base-export.service.ts
@@ -23,7 +23,7 @@ import type {
import archiver from 'archiver';
import { stringify } from 'csv-stringify/sync';
import { Knex } from 'knex';
-import { omit, pick } from 'lodash';
+import { groupBy, omit, pick } from 'lodash';
import { InjectModel } from 'nest-knexjs';
import { ClsService } from 'nestjs-cls';
import { IStorageConfig, StorageConfig } from '../../configs/storage';
@@ -47,6 +47,9 @@ import { EXCLUDE_SYSTEM_FIELDS } from './constant';
@Injectable()
export class BaseExportService {
public static CSV_CHUNK = 500;
+ public static FIELD_EXPORT_BATCH_SIZE = 500;
+ public static VIEW_EXPORT_BATCH_SIZE = 500;
+ private static readonly TABLE_ID_QUERY_CHUNK_SIZE = 100;
public static FILE_SUFFIX = 'tea';
public static EXPORT_FIELD_COLUMNS = [
'id',
@@ -336,25 +339,8 @@ export class BaseExportService {
},
});
const tableIds = tableRaws.map(({ id }) => id);
- const fieldRaws = await prisma.field.findMany({
- where: {
- tableId: {
- in: tableIds,
- },
- deletedTime: null,
- },
- });
- const viewRaws = await prisma.view.findMany({
- where: {
- tableId: {
- in: tableIds,
- },
- deletedTime: null,
- },
- orderBy: {
- order: 'asc',
- },
- });
+ const fieldRaws = await this.findFieldsByTableIds(tableIds);
+ const viewRaws = await this.findViewsByTableIds(tableIds);
// 2. generate base structure json
onProgress?.('exporting_structure');
@@ -504,6 +490,8 @@ export class BaseExportService {
fieldRaws,
destSpaceId
);
+ const fieldsByTableId = groupBy(fieldRaws, 'tableId');
+ const viewsByTableId = groupBy(viewRaws, 'tableId');
const tables = [] as IBaseJson['tables'];
for (const table of tableRaws) {
const { name, description, order, id, icon, dbTableName } = table;
@@ -516,18 +504,18 @@ export class BaseExportService {
icon,
dbTableName: realDbTableName,
} as IBaseJson['tables'][number];
- const currentTableFields = fieldRaws.filter(({ tableId }) => tableId === id);
+ const currentTableFields = fieldsByTableId[id] ?? [];
tableObject.fields = this.generateFieldConfig(
currentTableFields,
allowCrossBase,
excludedTableIds,
crossSpaceForeignBaseIds
);
- tableObject.views = this.generateViewConfig(viewRaws.filter(({ tableId }) => tableId === id));
+ tableObject.views = this.generateViewConfig(viewsByTableId[id] ?? []);
tables.push(tableObject);
}
- const plugins = await this.generatePluginConfig(baseId, includedDashboardIds);
+ const plugins = await this.generatePluginConfig(baseId, includedDashboardIds, viewRaws);
const folders = await this.generateFolderConfig(baseId, includedFolderIds);
const nodes = await this.generateNodeConfig(baseId, includeNodes, rootNodeIds);
@@ -919,6 +907,98 @@ export class BaseExportService {
return await prisma.$queryRawUnsafe[]>(recordsQuery);
}
+ async findFieldsByTableIds(tableIds: string[]) {
+ const prisma = this.prismaService.txClient();
+ const fieldRaws: Field[] = [];
+
+ for (
+ let index = 0;
+ index < tableIds.length;
+ index += BaseExportService.TABLE_ID_QUERY_CHUNK_SIZE
+ ) {
+ const tableIdChunk = tableIds.slice(
+ index,
+ index + BaseExportService.TABLE_ID_QUERY_CHUNK_SIZE
+ );
+ let cursor: string | undefined;
+ let hasMore = true;
+
+ while (hasMore) {
+ const page = await prisma.field.findMany({
+ where: {
+ tableId: { in: tableIdChunk },
+ deletedTime: null,
+ },
+ ...(cursor
+ ? {
+ cursor: {
+ id: cursor,
+ },
+ skip: 1,
+ }
+ : {}),
+ orderBy: [{ tableId: 'asc' }, { id: 'asc' }],
+ take: BaseExportService.FIELD_EXPORT_BATCH_SIZE,
+ });
+
+ fieldRaws.push(...page);
+
+ hasMore = page.length === BaseExportService.FIELD_EXPORT_BATCH_SIZE;
+ if (hasMore) {
+ cursor = page[page.length - 1].id;
+ }
+ }
+ }
+
+ return fieldRaws;
+ }
+
+ async findViewsByTableIds(tableIds: string[]) {
+ const prisma = this.prismaService.txClient();
+ const viewRaws: View[] = [];
+
+ for (
+ let index = 0;
+ index < tableIds.length;
+ index += BaseExportService.TABLE_ID_QUERY_CHUNK_SIZE
+ ) {
+ const tableIdChunk = tableIds.slice(
+ index,
+ index + BaseExportService.TABLE_ID_QUERY_CHUNK_SIZE
+ );
+ let cursor: string | undefined;
+ let hasMore = true;
+
+ while (hasMore) {
+ const page = await prisma.view.findMany({
+ where: {
+ tableId: { in: tableIdChunk },
+ deletedTime: null,
+ },
+ ...(cursor
+ ? {
+ cursor: {
+ id: cursor,
+ },
+ skip: 1,
+ }
+ : {}),
+ orderBy: [{ tableId: 'asc' }, { order: 'asc' }, { id: 'asc' }],
+ take: BaseExportService.VIEW_EXPORT_BATCH_SIZE,
+ });
+
+ viewRaws.push(...page);
+
+ hasMore = page.length === BaseExportService.VIEW_EXPORT_BATCH_SIZE;
+ if (hasMore) {
+ cursor = page[page.length - 1].id;
+ }
+ }
+ }
+
+ return viewRaws;
+ }
+
/**
* @description convert the cell value to the csv value
* @param value - the cell value
@@ -1437,7 +1517,7 @@ export class BaseExportService {
});
}
- async generatePluginConfig(baseId: string, includedDashboardIds?: string[]) {
+ async generatePluginConfig(baseId: string, includedDashboardIds?: string[], viewRaws?: View[]) {
const pluginJson = {} as IBaseJson['plugins'];
pluginJson[PluginPosition.Dashboard] = await this.generateDashboard(
@@ -1447,35 +1527,17 @@ export class BaseExportService {
pluginJson[PluginPosition.Panel] = await this.generatePluginPanel(baseId);
- pluginJson[PluginPosition.View] = await this.generatePluginView(baseId);
+ pluginJson[PluginPosition.View] = await this.generatePluginView(baseId, viewRaws);
return pluginJson;
}
- private async generatePluginView(baseId: string) {
- const tableIds = await this.prismaService.txClient().tableMeta.findMany({
- where: {
- baseId,
- deletedTime: null,
- },
- });
-
- const prisma = this.prismaService.txClient();
-
- const viewPluginRaws = await prisma.view.findMany({
- where: {
- tableId: {
- in: tableIds.map(({ id }) => id),
- },
- type: ViewType.Plugin,
- deletedTime: null,
- },
- orderBy: {
- createdTime: 'asc',
- },
- });
+ private async generatePluginView(baseId: string, viewRaws?: View[]) {
+ const viewPluginRaws = viewRaws
+ ? viewRaws.filter(({ type }) => type === ViewType.Plugin)
+ : await this.findPluginViewsByBaseId(baseId);
- const viewPluginInstallRaws = await prisma.pluginInstall.findMany({
+ const viewPluginInstallRaws = await this.prismaService.txClient().pluginInstall.findMany({
where: {
positionId: {
in: viewPluginRaws.map(({ id }) => id),
@@ -1503,6 +1565,22 @@ export class BaseExportService {
}) as unknown as IBaseJson['plugins'][PluginPosition.View];
}
+ private async findPluginViewsByBaseId(baseId: string) {
+ const tableIds = await this.prismaService.txClient().tableMeta.findMany({
+ where: {
+ baseId,
+ deletedTime: null,
+ },
+ select: {
+ id: true,
+ },
+ });
+
+ return (await this.findViewsByTableIds(tableIds.map(({ id }) => id))).filter(
+ ({ type }) => type === ViewType.Plugin
+ );
+ }
+
private async generatePluginPanel(baseId: string) {
const prisma = this.prismaService.txClient();
const tableIds = await prisma.tableMeta.findMany({
diff --git a/apps/nestjs-backend/src/features/base/base-freeze.service.spec.ts b/apps/nestjs-backend/src/features/base/base-freeze.service.spec.ts
new file mode 100644
index 0000000000..fb737fbf58
--- /dev/null
+++ b/apps/nestjs-backend/src/features/base/base-freeze.service.spec.ts
@@ -0,0 +1,67 @@
+import { HttpErrorCode } from '@teable/core';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { CustomHttpException } from '../../custom.exception';
+import { BaseService } from './base.service';
+
+describe('BaseService write freeze', () => {
+ const freezeError = new CustomHttpException(
+ 'Space data database migration is in progress',
+ HttpErrorCode.CONFLICT,
+ {
+ errorCode: 'SPACE_DATA_DB_MIGRATING',
+ migrationJobId: 'sdmjxxx',
+ }
+ );
+ const prismaService = {
+ base: {
+ create: vi.fn(),
+ update: vi.fn(),
+ },
+ };
+ const migrationGuard = {
+ assertSpaceWritable: vi.fn(),
+ assertBaseWritable: vi.fn(),
+ };
+
+ const service = () =>
+ new BaseService(
+ prismaService as never,
+ {} as never,
+ { get: vi.fn().mockReturnValue('usrxxx') } as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ migrationGuard as never
+ );
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ migrationGuard.assertSpaceWritable.mockRejectedValue(freezeError);
+ migrationGuard.assertBaseWritable.mockRejectedValue(freezeError);
+ });
+
+ it('rejects base creation before calculating order or creating metadata when the target space is migrating', async () => {
+ await expect(
+ service().createBase({ name: 'Blocked base', spaceId: 'spcxxx' } as never)
+ ).rejects.toBe(freezeError);
+
+ expect(migrationGuard.assertSpaceWritable).toHaveBeenCalledWith('spcxxx');
+ expect(prismaService.base.create).not.toHaveBeenCalled();
+ });
+
+ it('rejects base update and delete before metadata writes when the base space is migrating', async () => {
+ await expect(service().updateBase('bsexxx', { name: 'Blocked' })).rejects.toBe(freezeError);
+ await expect(service().deleteBase('bsexxx')).rejects.toBe(freezeError);
+
+ expect(migrationGuard.assertBaseWritable).toHaveBeenCalledTimes(2);
+ expect(prismaService.base.update).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts
index 8d41623670..a1b9ffa07c 100644
--- a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts
+++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.spec.ts
@@ -4,7 +4,7 @@ import { vi } from 'vitest';
import { BaseImportCsvQueueProcessor } from './base-import-csv.processor';
describe('BaseImportCsvQueueProcessor', () => {
- it('writes imported record history into the routed data DB internal schema', async () => {
+ it('does not write record history when importing base records', async () => {
const executedSql: string[] = [];
const dataKnex = Knex({ client: 'pg' });
const dataPrisma = {
@@ -64,6 +64,10 @@ describe('BaseImportCsvQueueProcessor', () => {
columnInfo: vi.fn().mockReturnValue('SELECT * FROM columns'),
};
processor.dataDbClientManager = dataDbClientManager;
+ Object.assign(processor, {
+ audit: { emitAtomic: vi.fn().mockResolvedValue(undefined) },
+ cls: { get: vi.fn() },
+ });
await processor.handleChunk(
[{ __id: 'recImported', fldText: 'Imported value' }],
@@ -82,10 +86,89 @@ describe('BaseImportCsvQueueProcessor', () => {
);
expect(executedSql.some((sql) => sql.includes('"bse_data"."tbl_imported"'))).toBe(true);
- expect(executedSql.some((sql) => sql.includes('"teable_internal"."record_history"'))).toBe(
- true
+ // Base import inserts records only — it must not generate per-cell record history
+ // (the v1 hot path that bloated each chunk transaction and timed out on large tables).
+ expect(executedSql.some((sql) => sql.toLowerCase().includes('record_history'))).toBe(false);
+
+ await dataKnex.destroy();
+ });
+
+ it('drops ghost columns missing from the target table instead of aborting the insert', async () => {
+ const executedSql: string[] = [];
+ const dataKnex = Knex({ client: 'pg' });
+ const dataPrisma = {
+ $queryRawUnsafe: vi
+ .fn()
+ // foreign keys info
+ .mockResolvedValueOnce([])
+ // columnInfo: the freshly-created table only has __id + fldText, NOT the ghost column
+ .mockResolvedValueOnce([{ name: '__id' }, { name: 'fldText' }]),
+ $executeRawUnsafe: vi.fn(async (sql: string) => {
+ executedSql.push(sql);
+ return 1;
+ }),
+ };
+ const prismaService = {
+ tableMeta: {
+ findUniqueOrThrow: vi.fn().mockResolvedValue({ dbTableName: 'bse_data.tbl_imported' }),
+ },
+ txClient: vi.fn().mockReturnValue({
+ attachmentsTable: { createMany: vi.fn().mockResolvedValue(undefined) },
+ }),
+ };
+ const dataDbClientManager = {
+ dataPrismaForBase: vi.fn().mockResolvedValue(dataPrisma),
+ dataKnexForBase: vi.fn().mockResolvedValue(dataKnex),
+ getDataDatabaseForBase: vi.fn().mockResolvedValue({
+ url: 'postgresql://user:pass@example.test:5432/data?schema=teable_internal',
+ }),
+ };
+ const processor = Object.create(BaseImportCsvQueueProcessor.prototype) as {
+ handleChunk: (
+ results: Record[],
+ config: Record,
+ excludeDbFieldNames: string[]
+ ) => Promise;
+ prismaService: typeof prismaService;
+ dbProvider: {
+ getForeignKeysInfo: ReturnType;
+ columnInfo: ReturnType;
+ };
+ dataDbClientManager: typeof dataDbClientManager;
+ };
+
+ processor.prismaService = prismaService;
+ processor.dbProvider = {
+ getForeignKeysInfo: vi.fn().mockReturnValue('SELECT * FROM foreign_keys'),
+ columnInfo: vi.fn().mockReturnValue('SELECT * FROM columns'),
+ };
+ processor.dataDbClientManager = dataDbClientManager;
+ Object.assign(processor, {
+ audit: { emitAtomic: vi.fn().mockResolvedValue(undefined) },
+ cls: { get: vi.fn() },
+ });
+
+ await processor.handleChunk(
+ // fldGhost is an orphan physical column dumped into the .tea CSV with no matching field
+ [{ __id: 'recImported', fldText: 'Imported value', fldGhost: 'orphan value' }],
+ {
+ baseId: 'bseImport',
+ tableId: 'tblImport',
+ userId: 'usrImport',
+ fieldIdMap: {},
+ viewIdMap: {},
+ fkMap: {},
+ attachmentsFields: [],
+ notNullFieldMap: new Map(),
+ fieldDbNameMap: new Map([['fldText', 'fldMappedText']]),
+ },
+ []
);
- expect(executedSql.some((sql) => sql.includes('insert into "record_history"'))).toBe(false);
+
+ const dataInsert = executedSql.find((sql) => sql.includes('"bse_data"."tbl_imported"'));
+ expect(dataInsert).toBeDefined();
+ expect(dataInsert).toContain('"fldText"');
+ expect(dataInsert).not.toContain('fldGhost');
await dataKnex.destroy();
});
diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts
index 84d8487977..07693ddb69 100644
--- a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts
+++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts
@@ -2,12 +2,7 @@
import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
import { Injectable, Logger } from '@nestjs/common';
import type { IAttachmentCellValue, ILinkFieldOptions } from '@teable/core';
-import {
- DbFieldType,
- FieldType,
- generateAttachmentId,
- generateRecordHistoryId,
-} from '@teable/core';
+import { DbFieldType, FieldType, generateAttachmentId } from '@teable/core';
import { PrismaService } from '@teable/db-main-prisma';
import type { IBaseJson, ImportBaseRo } from '@teable/openapi';
import { CreateRecordAction, UploadType } from '@teable/openapi';
@@ -200,9 +195,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost {
});
}
});
- const fieldDbNameMap = new Map(
- table?.fields?.map(({ dbFieldName, id }) => [dbFieldName, fieldIdMap[id] ?? id]) ?? []
- );
const batchProcessor = new BatchProcessor>(async (chunk) => {
totalRecordsCount += chunk.length;
@@ -219,7 +211,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost {
fkMap,
attachmentsFields,
notNullFieldMap,
- fieldDbNameMap,
},
excludeDbFieldNames
);
@@ -305,10 +296,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost {
return await fn(dataPrisma);
}
- private getDataDbInternalSchema(dataDbUrl: string) {
- return new URL(dataDbUrl).searchParams.get('schema') || 'public';
- }
-
// Raw SQL chunk insert (no v2 events). Active BaseImport operation is set by
// `handleBaseImportCsv` above; the `@Audit` atomic emit mode writes one audit row per
// chunk with atomic record-create action and rootAction=BaseImport.
@@ -327,20 +314,11 @@ export class BaseImportCsvQueueProcessor extends WorkerHost {
fkMap: Record;
attachmentsFields: { dbFieldName: string; id: string }[];
notNullFieldMap: Map;
- fieldDbNameMap: Map;
},
excludeDbFieldNames: string[]
) {
- const {
- baseId,
- tableId,
- userId,
- fieldIdMap,
- attachmentsFields,
- fkMap,
- notNullFieldMap,
- fieldDbNameMap,
- } = config;
+ const { baseId, tableId, userId, fieldIdMap, attachmentsFields, fkMap, notNullFieldMap } =
+ config;
const { dbTableName } = await this.prismaService.tableMeta.findUniqueOrThrow({
where: { id: tableId },
select: {
@@ -364,22 +342,11 @@ export class BaseImportCsvQueueProcessor extends WorkerHost {
recordId: string;
fieldId: string;
}[];
- const recordHistoryList: {
- id: string;
- table_id: string;
- record_id: string;
- field_id: string;
- before: string;
- after: string;
- created_by: string;
- }[] = [];
const dataPrisma = (await this.dataDbClientManager.dataPrismaForBase(
baseId
)) as IDataPrismaScopedClient;
const dataKnex = await this.dataDbClientManager.dataKnexForBase(baseId);
- const dataDb = await this.dataDbClientManager.getDataDatabaseForBase(baseId);
- const dataDbInternalSchema = this.getDataDbInternalSchema(dataDb.url);
await this.dataTransaction(dataPrisma, async (prisma) => {
// delete foreign keys if(exist) then duplicate table data
@@ -432,9 +399,24 @@ export class BaseImportCsvQueueProcessor extends WorkerHost {
: fkMap[name] || name;
});
+ // Authoritative set of columns that actually exist on the freshly-created table.
+ // The .tea CSV is dumped from the source data table's physical columns, which can
+ // include "ghost" columns left behind by deleted/renamed fields. Those have no field
+ // in the exported structure, so the new table lacks them; inserting them raw aborts the
+ // whole table transaction (42703 -> 25P02) and drops every record. Mirror the v2 import,
+ // which only restores columns present in the field metadata.
+ const realColumns = new Set(columnInfo.map(({ name }) => name));
+
const recordsToInsert = newResult.map((result) => {
const res = { ...result };
Object.entries(res).forEach(([key, value]) => {
+ // drop ghost business columns absent from the target table (system / __fk_ / __row_
+ // columns are left to the dedicated handling below and the lacking-column ALTER step)
+ if (!key.startsWith('__') && !realColumns.has(key)) {
+ delete res[key];
+ return;
+ }
+
if (res[key] === '') {
const notNullInfo = notNullFieldMap.get(key);
if (notNullInfo) {
@@ -468,24 +450,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost {
});
});
}
-
- if (key.startsWith('__') && !key.startsWith('__fk_')) {
- return;
- }
-
- const sourceFieldId = key.startsWith('__fk_') ? key.slice(5) : key;
- const fieldId = fieldIdMap[sourceFieldId] ?? fieldDbNameMap.get(key) ?? sourceFieldId;
- if (fieldId && value !== '' && value != null) {
- recordHistoryList.push({
- id: generateRecordHistoryId(),
- table_id: tableId,
- record_id: res['__id'] as string,
- field_id: fieldId,
- before: JSON.stringify({ data: null }),
- after: JSON.stringify({ data: value }),
- created_by: userId,
- });
- }
});
// default value set
@@ -513,15 +477,6 @@ export class BaseImportCsvQueueProcessor extends WorkerHost {
const sql = dataKnex.table(dbTableName).insert(recordsToInsert).toQuery();
await prisma.$executeRawUnsafe(sql);
-
- if (recordHistoryList.length) {
- const historySql = dataKnex
- .withSchema(dataDbInternalSchema)
- .insert(recordHistoryList)
- .into('record_history')
- .toQuery();
- await prisma.$executeRawUnsafe(historySql);
- }
});
// restore foreign keys with NOT VALID
diff --git a/apps/nestjs-backend/src/features/base/base-import.service.spec.ts b/apps/nestjs-backend/src/features/base/base-import.service.spec.ts
index 45deeffa89..89e61a8691 100644
--- a/apps/nestjs-backend/src/features/base/base-import.service.spec.ts
+++ b/apps/nestjs-backend/src/features/base/base-import.service.spec.ts
@@ -1,9 +1,10 @@
import type { Readable } from 'stream';
-import { DbFieldType, FieldType } from '@teable/core';
-import type { IBaseJson, ImportBaseRo } from '@teable/openapi';
+import { DbFieldType, FieldType, HttpErrorCode } from '@teable/core';
+import { BaseDuplicateMode, type IBaseJson, type ImportBaseRo } from '@teable/openapi';
import type { RestoreRecordInput } from '@teable/v2-core';
import archiver from 'archiver';
import { vi } from 'vitest';
+import { CustomHttpException } from '../../custom.exception';
import { BaseImportService, formatBaseImportError } from './base-import.service';
@@ -39,10 +40,29 @@ interface IProcessStructureService {
createBaseStructure: ReturnType;
}
+interface IImportBaseV2Service {
+ importBaseV2(
+ importBaseRo: Pick,
+ onProgress?: (...args: unknown[]) => void
+ ): Promise;
+ storageAdapter: unknown;
+ readDotTeaStructure: ReturnType;
+ v2ContainerService: unknown;
+ v2ContextFactory: unknown;
+ createBaseV2: ReturnType;
+ restoreBaseExtrasV2: ReturnType;
+ importAttachmentsV2: ReturnType;
+ importTableDataV2: ReturnType;
+ importTableLinkFieldsV2: ReturnType;
+ audit: unknown;
+ cls: unknown;
+}
+
const dbTableName = 'bse_test.tbl_test';
const jsonColumnName = 'json_col';
const textColumnName = 'text_col';
const textCellValue = 'plain text';
+const importedBaseName = 'Imported base';
const createService = () =>
Object.create(BaseImportService.prototype) as IRestoreRecordInputBuilder;
@@ -55,6 +75,15 @@ const createZipStream = (structure: IBaseJson) => {
};
describe('BaseImportService', () => {
+ const freezeError = new CustomHttpException(
+ 'Space data database migration is in progress',
+ HttpErrorCode.CONFLICT,
+ {
+ errorCode: 'SPACE_DATA_DB_MIGRATING',
+ migrationJobId: 'sdmjxxx',
+ }
+ );
+
describe('formatBaseImportError', () => {
it('falls back when an Error has an empty message', () => {
expect(formatBaseImportError(new Error(''), 'Unknown import error')).toBe(
@@ -84,6 +113,43 @@ describe('BaseImportService', () => {
});
describe('processStructure', () => {
+ it('rejects import before downloading files or opening a transaction when the space is migrating', async () => {
+ const service = Object.create(BaseImportService.prototype) as {
+ importBase: BaseImportService['importBase'];
+ importBaseV2: BaseImportService['importBaseV2'];
+ audit: { withOperation: ReturnType };
+ cls: { get: ReturnType };
+ spaceDataDbMigrationGuard: { assertSpaceWritable: ReturnType };
+ storageAdapter: { downloadFile: ReturnType };
+ prismaService: { $tx: ReturnType };
+ };
+ service.audit = {
+ withOperation: vi.fn((_, fn: () => Promise) => fn()),
+ };
+ service.cls = { get: vi.fn() };
+ service.spaceDataDbMigrationGuard = {
+ assertSpaceWritable: vi.fn().mockRejectedValue(freezeError),
+ };
+ service.storageAdapter = { downloadFile: vi.fn() };
+ service.prismaService = { $tx: vi.fn() };
+ const importRo = {
+ spaceId: 'spcImport',
+ notify: { path: 'imports/base.tea' },
+ } as ImportBaseRo;
+ const onProgress = vi.fn();
+
+ await expect(service.importBase(importRo, onProgress)).rejects.toBe(freezeError);
+ await expect(service.importBaseV2(importRo, onProgress)).rejects.toBe(freezeError);
+
+ expect(service.spaceDataDbMigrationGuard.assertSpaceWritable).toHaveBeenCalledTimes(2);
+ expect(service.spaceDataDbMigrationGuard.assertSpaceWritable).toHaveBeenCalledWith(
+ 'spcImport'
+ );
+ expect(onProgress).not.toHaveBeenCalled();
+ expect(service.storageAdapter.downloadFile).not.toHaveBeenCalled();
+ expect(service.prismaService.$tx).not.toHaveBeenCalled();
+ });
+
it('passes transaction-aware data DB routing into structure creation', async () => {
const service = Object.create(BaseImportService.prototype) as IProcessStructureService;
const structure = {
@@ -123,14 +189,14 @@ describe('BaseImportService', () => {
it('creates imported base schemas through the space routed data client', async () => {
const createdBase = {
id: 'bseImported',
- name: 'Imported base',
+ name: importedBaseName,
spaceId: 'spcImport',
order: 1,
};
const baseCreate = vi.fn().mockResolvedValue(createdBase);
const baseUpdate = vi.fn().mockResolvedValue({
...createdBase,
- name: 'Imported base',
+ name: importedBaseName,
});
const routedExecute = vi.fn().mockResolvedValue(0);
const fallbackExecute = vi.fn().mockResolvedValue(0);
@@ -145,7 +211,9 @@ describe('BaseImportService', () => {
cls: unknown;
prismaService: unknown;
dbProvider: unknown;
- dataDbClientManager: unknown;
+ dataDbClientManager: {
+ dataPrismaForSpace: ReturnType;
+ };
dataPrismaService: unknown;
};
@@ -174,10 +242,10 @@ describe('BaseImportService', () => {
};
await expect(
- service.createBase('spcImport', 'Imported base', 'icon', { useTransaction: true })
+ service.createBase('spcImport', importedBaseName, 'icon', { useTransaction: true })
).resolves.toMatchObject({
id: 'bseImported',
- name: 'Imported base',
+ name: importedBaseName,
});
expect(service.dataDbClientManager.dataPrismaForSpace).toHaveBeenCalledWith('spcImport', {
@@ -188,6 +256,101 @@ describe('BaseImportService', () => {
});
});
+ describe('importBaseV2', () => {
+ it('restores edition extras during dottea import and returns their id maps', async () => {
+ const tableIdMap = { tblSource: 'tblImported' };
+ const fieldIdMap = { fldSource: 'fldImported' };
+ const viewIdMap = { viwSource: 'viwImported' };
+ const appIdMap = { appSource: 'appImported' };
+ const workflowIdMap = { wflSource: 'wflImported' };
+ const commandBus = {
+ execute: vi.fn().mockResolvedValue({
+ isErr: () => false,
+ value: {
+ tableIdMap,
+ fieldIdMap,
+ viewIdMap,
+ },
+ }),
+ };
+ const queryBus = {};
+ const tableRecordRepository = {};
+ const unitOfWork = {};
+ const db = {};
+ const context = {};
+ const container = {
+ resolve: vi.fn((token: unknown) => {
+ const tokenText = String(token);
+ if (tokenText.includes('commandBus')) return commandBus;
+ if (tokenText.includes('queryBus')) return queryBus;
+ if (tokenText.includes('tableRecordRepository')) return tableRecordRepository;
+ if (tokenText.includes('unitOfWork')) return unitOfWork;
+ return db;
+ }),
+ };
+ const structure = {
+ id: 'bseSource',
+ name: 'Source base',
+ icon: 'icon',
+ tables: [],
+ plugins: {},
+ folders: [],
+ nodes: [],
+ } as unknown as IBaseJson;
+ const importedBase = { id: 'bseImported', name: 'Source base', spaceId: 'spcImport' };
+ const service = Object.create(BaseImportService.prototype) as IImportBaseV2Service;
+
+ service.storageAdapter = {
+ downloadFile: vi.fn().mockReturnValue({}),
+ };
+ service.readDotTeaStructure = vi.fn().mockResolvedValue(structure);
+ service.v2ContainerService = {
+ getContainerForSpace: vi.fn().mockResolvedValue(container),
+ };
+ service.v2ContextFactory = {
+ createContext: vi.fn().mockResolvedValue(context),
+ };
+ service.createBaseV2 = vi.fn().mockResolvedValue(importedBase);
+ service.restoreBaseExtrasV2 = vi.fn().mockResolvedValue({ appIdMap, workflowIdMap });
+ service.importAttachmentsV2 = vi.fn().mockResolvedValue(undefined);
+ service.importTableDataV2 = vi.fn().mockResolvedValue(undefined);
+ service.importTableLinkFieldsV2 = vi.fn().mockResolvedValue(undefined);
+ service.audit = {
+ withOperation: vi.fn((_resolved, run: () => Promise) => run()),
+ };
+ service.cls = {
+ get: vi.fn().mockReturnValue('usrImport'),
+ };
+
+ await expect(
+ service.importBaseV2({
+ spaceId: 'spcImport',
+ notify: { path: 'import.tea' } as ImportBaseRo['notify'],
+ })
+ ).resolves.toMatchObject({
+ base: importedBase,
+ tableIdMap,
+ fieldIdMap,
+ viewIdMap,
+ appIdMap,
+ workflowIdMap,
+ baseIdMap: { [structure.id]: importedBase.id },
+ });
+
+ expect(service.restoreBaseExtrasV2).toHaveBeenCalledWith(
+ db,
+ importedBase.id,
+ structure,
+ { tableIdMap, fieldIdMap, viewIdMap },
+ BaseDuplicateMode.Normal,
+ undefined
+ );
+ expect(commandBus.execute).toHaveBeenCalledTimes(1);
+ expect(service.importTableDataV2).toHaveBeenCalledTimes(1);
+ expect(service.importTableLinkFieldsV2).toHaveBeenCalledTimes(1);
+ });
+ });
+
describe('toRestoreRecordInput', () => {
it('serializes JSON extra column values for v2 dottea row restore', () => {
const service = createService();
diff --git a/apps/nestjs-backend/src/features/base/base-import.service.ts b/apps/nestjs-backend/src/features/base/base-import.service.ts
index 6c6efeec85..cbe4673903 100644
--- a/apps/nestjs-backend/src/features/base/base-import.service.ts
+++ b/apps/nestjs-backend/src/features/base/base-import.service.ts
@@ -1,5 +1,5 @@
import type { Readable } from 'stream';
-import { Injectable, Logger } from '@nestjs/common';
+import { Inject, Injectable, Logger, Optional } from '@nestjs/common';
import { EventEmitter2 } from '@nestjs/event-emitter';
import {
DbFieldType,
@@ -76,6 +76,7 @@ import { InjectStorageAdapter } from '../attachments/plugins/storage';
import { AuditScope } from '../audit/audit-scope';
import { Audit } from '../audit/audit.decorator';
import { FieldDuplicateService } from '../field/field-duplicate/field-duplicate.service';
+import { SpaceDataDbMigrationGuardService } from '../space/space-data-db-migration-guard.service';
import { TableService } from '../table/table.service';
import { V2ContainerService } from '../v2/v2-container.service';
import { V2ExecutionContextFactory } from '../v2/v2-execution-context.factory';
@@ -112,6 +113,7 @@ type IDataPrismaScopedClient = IDataPrismaExecutor & {
const tableDataImportBatchSize = 100;
const linkFieldImportBatchSize = 25;
+const attachmentsDirPrefix = 'attachments/';
const stringifyErrorDetails = (details: unknown): string | undefined => {
if (details === undefined || details === null) {
@@ -181,9 +183,20 @@ export class BaseImportService {
private readonly v2ContainerService: V2ContainerService,
private readonly v2ContextFactory: V2ExecutionContextFactory,
private readonly dataDbClientManager: DataDbClientManager,
- private readonly audit: AuditScope
+ private readonly audit: AuditScope,
+ @Optional()
+ @Inject(SpaceDataDbMigrationGuardService)
+ private readonly spaceDataDbMigrationGuard?: SpaceDataDbMigrationGuardService
) {}
+ private async assertSpaceWritable(spaceId: string) {
+ await this.spaceDataDbMigrationGuard?.assertSpaceWritable(spaceId);
+ }
+
+ private async assertBaseWritable(baseId: string) {
+ await this.spaceDataDbMigrationGuard?.assertBaseWritable(baseId);
+ }
+
private async getMaxOrder(spaceId: string) {
const spaceAggregate = await this.prismaService.txClient().base.aggregate({
where: { spaceId, deletedTime: null },
@@ -198,6 +211,7 @@ export class BaseImportService {
icon?: string,
routingOptions?: IDataDbRoutingOptions
) {
+ await this.assertSpaceWritable(spaceId);
const userId = this.cls.get('user.id');
const order = (await this.getMaxOrder(spaceId)) + 1;
@@ -259,7 +273,9 @@ export class BaseImportService {
updateExistingBase: boolean = true
): Promise {
const userId = this.cls.get('user.id');
+ await this.assertSpaceWritable(spaceId);
if (baseId) {
+ await this.assertBaseWritable(baseId);
const existingResult = await sql<{
id: string;
name: string;
@@ -351,8 +367,10 @@ export class BaseImportService {
})
async importBase(importBaseRo: ImportBaseRo, onProgress?: BaseImportProgressCallback) {
const {
+ spaceId,
notify: { path },
} = importBaseRo;
+ await this.assertSpaceWritable(spaceId);
const logId = this.audit.current()!.operationId;
onProgress?.('parsing_structure');
@@ -418,6 +436,7 @@ export class BaseImportService {
spaceId,
notify: { path },
} = importBaseRo;
+ await this.assertSpaceWritable(spaceId);
onProgress?.('importing_v2');
onProgress?.('parsing_structure');
@@ -473,7 +492,7 @@ export class BaseImportService {
const { tableIdMap, fieldIdMap, viewIdMap } = result.value;
onProgress?.('structure_created', base.id);
- await this.restoreBaseExtrasV2(
+ const { appIdMap, workflowIdMap } = await this.restoreBaseExtrasV2(
db,
base.id,
structure,
@@ -512,6 +531,15 @@ export class BaseImportService {
tableIdMap,
fieldIdMap,
viewIdMap,
+ appIdMap,
+ workflowIdMap,
+ // Source->new base id, so EE import can rewrite base-id references baked into app
+ // build artifacts (matches the v1 import/duplicate idMap; v2 dropped it).
+ baseIdMap: { [structure.id]: base.id },
+ } as IImportBaseVo & {
+ appIdMap: Record;
+ workflowIdMap: Record;
+ baseIdMap: Record;
};
}
@@ -525,8 +553,7 @@ export class BaseImportService {
viewIdMap: Record;
},
duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal,
- onProgress?: BaseImportProgressCallback,
- options?: { restoreEeResources?: boolean }
+ onProgress?: BaseImportProgressCallback
): Promise<{ appIdMap: Record