feat(file-manager): add AI image tagging feature

packages/api-file-manager/package.json

@@ -22,6 +22,7 @@
     "@webiny/api": "0.0.0",
     "@webiny/api-core": "0.0.0",
     "@webiny/api-headless-cms": "0.0.0",
+    "@webiny/api-websockets": "0.0.0",
     "@webiny/aws-sdk": "0.0.0",
     "@webiny/build-tools": "0.0.0",
     "@webiny/error": "0.0.0",
@@ -36,6 +37,7 @@
     "cache-control-parser": "^2.0.6",
     "lodash": "^4.17.21",
     "object-hash": "^3.0.0",
+    "openai": "^6.10.0",
     "zod": "^3.25.76"
   },
   "devDependencies": {

packages/api-file-manager/src/index.ts

@@ -7,6 +7,7 @@ import { createGraphQLSchemaPlugin } from "./graphql/index.js";
 import { applyThreatScanning } from "./enterprise/applyThreatScanning.js";
 import type { FileManagerConfig } from "./createFileManager/types.js";
 import { FileManagerFeature } from "~/features/FileManagerFeature.js";
+import { createFileTaggingTask } from "./tasks/createFileTaggingTask.js";
 
 export * from "./modelModifier/CmsModelModifier.js";
 export * from "./plugins/index.js";
@@ -15,7 +16,7 @@ export * from "./delivery/index.js";
 export const createFileManagerContext = ({
     storageOperations
 }: Pick<FileManagerConfig, "storageOperations">) => {
-    const plugin = new ContextPlugin<FileManagerContext>(async context => {
+    const fmContextPlugin = new ContextPlugin<FileManagerContext>(async context => {
         const fmContext = new FileManagerContextSetup(context);
         context.fileManager = await fmContext.setupContext(storageOperations);
 
@@ -26,9 +27,25 @@ export const createFileManagerContext = ({
         FileManagerFeature.register(context.container);
     });
 
-    plugin.name = "file-manager.createContext";
-
-    return plugin;
+    fmContextPlugin.name = "file-manager.createContext";
+
+    if (process.env.WEBINY_API_AI_IMAGE_TAGGING === "true") {
+        // Trigger background task
+        const aiImageTaggingPlugin = new ContextPlugin<FileManagerContext>(context => {
+            context.fileManager.onFileAfterCreate.subscribe(({ file }) => {
+                context.tasks.trigger({
+                    definition: "fmAiImageTagging",
+                    input: {
+                        file: file
+                    },
+                    name: "AI Image Tagging Task"
+                });
+            });
+        });
+        return [fmContextPlugin, createFileTaggingTask(), aiImageTaggingPlugin];
+    }
+
+    return [fmContextPlugin];
 };
 
 export const createFileManagerGraphQL = () => {

packages/api-file-manager/src/tasks/createFileTaggingTask.ts

@@ -0,0 +1,127 @@
+import { createTaskDefinition } from "@webiny/tasks";
+import type { FileManagerContext } from "~/types.js";
+import { Context as WebsocketsContext } from "@webiny/api-websockets";
+import OpenAI from "openai";
+
+interface AITaggingResponse {
+    tags: string[];
+}
+
+async function getTagsFromAI(imageUrl: string): Promise<AITaggingResponse> {
+    const apiKey = process.env.WEBINY_API_OPEN_AI_API_KEY;
+
+    if (!apiKey) {
+        throw new Error("WEBINY_API_OPEN_AI_API_KEY environment variable is not set");
+    }
+
+    const openai = new OpenAI({
+        apiKey: apiKey
+    });
+
+    const prompt = `You are an AI trained to visually analyze real images (not filenames or metadata) and generate concise, human-friendly metadata for image organization and accessibility.
+
+    Task: Visually Analyze the image from the provided public URL. Do not create or modify any image.
+
+    Given this image URL, return:
+      1. Tags (5 maximum) —
+        - Single words or short phrases only.
+        - Avoid duplicates or overly specific variations.
+        - Reflect the main subjects, mood, setting, or concept of the image.
+        - Use lowercase words separated by commas.
+      2. Alt Text (Caption) —
+        - One clear, natural-sounding sentence (under 25 words).
+        - Describe what a person would perceive in the image without interpretation or exaggeration.
+        - Avoid starting with "Image of" or "Picture of."
+
+    Output Format:
+    Tags: [tag1, tag2, tag3, tag4, tag5]
+    Alt Text: "Your caption here."`;
+
+    const completion = await openai.chat.completions.create({
+        model: "gpt-4o",
+        messages: [
+            {
+                role: "user",
+                content: [
+                    { type: "text", text: prompt },
+                    {
+                        type: "image_url",
+                        image_url: {
+                            url: imageUrl
+                        }
+                    }
+                ]
+            }
+        ],
+        max_tokens: 300
+    });
+
+    const responseText = completion.choices[0]?.message?.content || "";
+
+    // Parse the response
+    const tagsMatch = responseText.match(/Tags:\s*\[(.*?)\]/);
+
+    const tags = tagsMatch
+        ? tagsMatch[1]
+              .split(",")
+              .map(tag => tag.trim())
+              .filter(tag => tag.length > 0)
+        : [];
+
+    return {
+        tags
+    };
+}
+
+export const createFileTaggingTask = () => {
+    return createTaskDefinition<FileManagerContext & WebsocketsContext>({
+        id: "fmAiImageTagging",
+        title: "File Upload Background task",
+        isPrivate: true,
+        async run(params) {
+            const { input, response, context } = params;
+
+            // Get the file to access its URL
+            const file = await context.fileManager.getFile(input.fileId);
+
+            if (!file) {
+                return response.error("File not found");
+            }
+
+            // Get tag information from AI
+            let tags: string[] = [];
+
+            try {
+                const settings = await context.fileManager.getSettings();
+                const fileURL = (settings?.srcPrefix || "") + file.key;
+
+                const aiResponse = await getTagsFromAI(fileURL);
+                tags = aiResponse.tags;
+            } catch (error) {
+                console.error("Error getting tags from AI:", error);
+                return response.error(`Failed to get tags from AI: ${error.message}`);
+            }
+
+            // Update File Tags
+            await context.fileManager.updateFile(input.fileId, {
+                tags: tags
+            });
+
+            // Send message to WebSocket
+            const allConnections = await context.websockets.listConnections();
+
+            await context.websockets.sendToConnections(allConnections, {
+                action: "fm.file.tags",
+                data: {
+                    tags,
+                    id: input.fileId
+                }
+            });
+
+            return response.done(
+                "successfully ran the fmAiImageTagging background task",
+                input.fileId
+            );
+        }
+    });
+};

packages/api-file-manager/tsconfig.build.json

@@ -5,6 +5,7 @@
     { "path": "../api/tsconfig.build.json" },
     { "path": "../api-core/tsconfig.build.json" },
     { "path": "../api-headless-cms/tsconfig.build.json" },
+    { "path":  "../api-websockets/tsconfig.build.json"},
     { "path": "../aws-sdk/tsconfig.build.json" },
     { "path": "../error/tsconfig.build.json" },
     { "path": "../feature/tsconfig.build.json" },

packages/api-file-manager/tsconfig.json

@@ -5,6 +5,7 @@
     { "path": "../api" },
     { "path": "../api-core" },
     { "path": "../api-headless-cms" },
+    { "path":  "../api-websockets"},
     { "path": "../aws-sdk" },
     { "path": "../error" },
     { "path": "../feature" },

packages/app-file-manager/src/modules/Enterprise/HandleWebsocketMessages.tsx

@@ -34,6 +34,14 @@ interface ThreatScan_UnsupportedFile extends IncomingGenericData {
     };
 }
 
+interface AITags extends IncomingGenericData {
+    action: "fm.file.tags";
+    data: {
+        id: string;
+        tags: string[];
+    };
+}
+
 export const HandleWebsocketMessages = () => {
     const { showErrorSnackbar } = useSnackbar();
     const websockets = useWebsockets();
@@ -89,10 +97,19 @@ export const HandleWebsocketMessages = () => {
             }
         );
 
+        const aiTags = websockets.onMessage<AITags>("fm.file.tags", async message => {
+            const { id, ...data } = message.data;
+
+            await fmViewRef.current.updateFile(id, data, {
+                localUpdate: true
+            });
+        });
+
         return () => {
             noThreat.off();
             threatDetected.off();
             unsupported.off();
+            aiTags.off();
         };
     }, []);
 

yarn.lock

@@ -14109,6 +14109,7 @@ __metadata:
     "@webiny/api": "npm:0.0.0"
     "@webiny/api-core": "npm:0.0.0"
     "@webiny/api-headless-cms": "npm:0.0.0"
+    "@webiny/api-websockets": "npm:0.0.0"
     "@webiny/aws-sdk": "npm:0.0.0"
     "@webiny/build-tools": "npm:0.0.0"
     "@webiny/error": "npm:0.0.0"
@@ -14125,6 +14126,7 @@ __metadata:
     cache-control-parser: "npm:^2.0.6"
     lodash: "npm:^4.17.21"
     object-hash: "npm:^3.0.0"
+    openai: "npm:^6.10.0"
     rimraf: "npm:^6.0.1"
     typescript: "npm:5.9.3"
     zod: "npm:^3.25.76"
@@ -28457,6 +28459,23 @@ __metadata:
   languageName: node
   linkType: hard
 
+"openai@npm:^6.10.0":
+  version: 6.10.0
+  resolution: "openai@npm:6.10.0"
+  peerDependencies:
+    ws: ^8.18.0
+    zod: ^3.25 || ^4.0
+  peerDependenciesMeta:
+    ws:
+      optional: true
+    zod:
+      optional: true
+  bin:
+    openai: bin/cli
+  checksum: 10/0e2c6a297337447aabbf1c7fb3f682abf41f8b80c4346479abb02462cfc0a32918dcb872bc6a92f4398c750f297a862b9aa783aa92f822149ceaee3983c83106
+  languageName: node
+  linkType: hard
+
 "opencollective-postinstall@npm:^2.0.2":
   version: 2.0.3
   resolution: "opencollective-postinstall@npm:2.0.3"