[sync] fix: refactor email processing and attachment handling (#1507) (#2824)

Synced from teableio/teable-ee@6177977

Co-authored-by: Uno <uno@teable.ai>

Author: tea-artist

apps/nestjs-backend/src/cache/types.ts

@@ -35,6 +35,7 @@ export interface ICacheStore {
   [key: `send-mail-rate-limit:${string}`]: boolean;
   [key: `oauth:token-rate:${string}:${string}`]: number;
   [key: `automation:email:rate:${string}:${number}`]: number;
+  [key: `automation:email-att:${string}`]: string[];
   // Distributed lock keys
   [key: `lock:${string}`]: string;
   [key: `import:result:manifest:${string}`]: {

apps/nestjs-backend/src/features/attachments/attachments.service.ts

@@ -33,7 +33,7 @@ import { AttachmentsStorageService } from './attachments-storage.service';
 import StorageAdapter from './plugins/adapter';
 import type { LocalStorage } from './plugins/local';
 import { InjectStorageAdapter } from './plugins/storage';
-import { getExtensionPreview } from './utils';
+import { getExtensionPreview, getSafeUploadContentType } from './utils';
 @Injectable()
 export class AttachmentsService {
   private logger = new Logger(AttachmentsService.name);
@@ -281,55 +281,6 @@ export class AttachmentsService {
     return await this.notifyToAttachmentItem(token, filename);
   }
 
-  async uploadFromLocalFile(filePath: string, filename: string): Promise<IAttachmentItem> {
-    const MAX_FILE_SIZE = this.thresholdConfig.maxOpenapiAttachmentUploadSize;
-    const stat = await fs.promises.stat(filePath);
-    const contentLength = stat.size;
-
-    if (contentLength > MAX_FILE_SIZE) {
-      const maxSize = (MAX_FILE_SIZE / (1024 * 1024)).toFixed(2);
-      throw new CustomHttpException(
-        `File size exceeds the maximum limit of ${maxSize} MB`,
-        HttpErrorCode.VALIDATION_ERROR,
-        {
-          localization: {
-            i18nKey: 'httpErrors.attachment.fileSizeExceedsMaximumLimit',
-            context: {
-              maxSize: `${maxSize}MB`,
-            },
-          },
-        }
-      );
-    }
-
-    const contentType = mimeTypes.lookup(filename) || 'application/octet-stream';
-    const { token, url } = await this.signature({
-      type: UploadType.Table,
-      contentLength,
-      contentType,
-      internal: true,
-    });
-
-    try {
-      await this.uploadStreamToStorage(
-        url,
-        fs.createReadStream(filePath),
-        contentType,
-        contentLength
-      );
-      return await this.notifyToAttachmentItem(token, filename);
-    } finally {
-      await fs.promises.unlink(filePath);
-      // Clean up temp subdirectory (created by email attachment saver)
-      const dir = dirname(filePath);
-      try {
-        await fs.promises.rmdir(dir);
-      } catch {
-        /* directory not empty or already removed */
-      }
-    }
-  }
-
   async uploadFromUrl(
     fileUrl: string,
     uploadType: UploadType = UploadType.Table
@@ -394,7 +345,10 @@ export class AttachmentsService {
       const headResponse = await axios.head(fileUrl);
       contentLength =
         headResponse.headers['content-length'] && parseInt(headResponse.headers['content-length']);
-      contentType = mimeTypes.lookup(fileUrl) || headResponse.headers['content-type'];
+      contentType =
+        mimeTypes.lookup(fileUrl) ||
+        headResponse.headers['content-type'] ||
+        'application/octet-stream';
       this.logger.log(
         `HEAD request successful. Content-Length: ${contentLength}, Content-Type: ${contentType}`
       );
@@ -460,7 +414,7 @@ export class AttachmentsService {
     try {
       await axios.put(url, stream, {
         headers: {
-          'Content-Type': contentType,
+          'Content-Type': getSafeUploadContentType(contentType),
           'Content-Length': contentLength,
         },
       });
@@ -473,7 +427,12 @@ export class AttachmentsService {
   private getFilenameFromUrl(url: string): string {
     const urlParts = new URL(url);
     const pathParts = urlParts.pathname.split('/');
-    return pathParts[pathParts.length - 1] || 'downloaded_file';
+    const rawFilename = pathParts[pathParts.length - 1] || 'downloaded_file';
+    try {
+      return decodeURIComponent(rawFilename);
+    } catch {
+      return rawFilename;
+    }
   }
 
   private async downloadFile(

apps/nestjs-backend/src/features/attachments/plugins/local.ts

@@ -18,6 +18,7 @@ import type { IClsStore } from '../../../types/cls';
 import { FileUtils } from '../../../utils';
 import { Encryptor } from '../../../utils/encryptor';
 import { second } from '../../../utils/second';
+import { isBodyParserFallback } from '../utils';
 import StorageAdapter from './adapter';
 import type { ILocalFileUpload, IObjectMeta, IPresignParams, IRespHeaders } from './types';
 
@@ -132,7 +133,7 @@ export class LocalStorage implements StorageAdapter {
         },
       });
     }
-    if (mimetype && mimetype !== contentType) {
+    if (mimetype && !isBodyParserFallback(mimetype, contentType) && mimetype !== contentType) {
       throw new CustomHttpException(
         `Not allow upload ${mimetype} file`,
         HttpErrorCode.VALIDATION_ERROR,

apps/nestjs-backend/src/features/attachments/utils.ts

@@ -1,3 +1,27 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+const OCTET_STREAM = 'application/octet-stream';
+const JSON_PREFIX = 'application/json';
+
+/**
+ * Check if a content type would be intercepted by Express body parser (e.g. application/json).
+ * When uploading internally via localhost, these types cause the stream to be consumed
+ * before reaching the upload handler, so we need to fall back to application/octet-stream.
+ */
+export const getSafeUploadContentType = (contentType: string): string => {
+  if (contentType && contentType.startsWith(JSON_PREFIX)) {
+    return OCTET_STREAM;
+  }
+  return contentType;
+};
+
+/**
+ * Check if a mimetype mismatch is caused by the body parser fallback.
+ * Returns true if the request used octet-stream as a substitute for a JSON content type.
+ */
+export const isBodyParserFallback = (mimetype: string, expectedType: string): boolean => {
+  return mimetype === OCTET_STREAM && expectedType.startsWith(JSON_PREFIX);
+};
+
 export const getExtensionPreview = (contentType: string) => {
   const imageExtensions = [
     'jif',