Merge pull request #197 from llm-tools/confluence_updates

Added image loader

Author: adhityan

docs/components/data-sources/image.mdx

@@ -0,0 +1,31 @@
+---
+title: '🖼️ Image'
+---
+
+You can load most images using an LLM to describe the contents of the image. This LLM by default is the one used with the application, 
+but a special LLM can also be specified for the content description. To load images, follow the steps below.
+
+## Install Image addon
+
+```bash
+npm install @llm-tools/embedjs-loader-image
+```
+
+## Usage
+
+### Load from a local file
+
+```ts
+import { RAGApplicationBuilder } from '@llm-tools/embedjs';
+import { ImageLoader } from '@llm-tools/embedjs-loader-image';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
+import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
+
+const app = await new RAGApplicationBuilder()
+.setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
+.setEmbeddingModel(new OpenAiEmbeddings())
+.setVectorDatabase(new HNSWDb())
+.build();
+
+app.addLoader(new ImageLoader({ filePathOrUrl: '/path/to/file.jpeg' }))
+```

docs/components/data-sources/overview.mdx

@@ -22,6 +22,7 @@ We handle the complexity of loading unstructured data from these data sources, a
     <Card title="Excel file" href="/components/data-sources/excel" />
     <Card title="Sitemap" href="/components/data-sources/sitemap" />
     <Card title="Markdown / MDX" href="/components/data-sources/markdown" />
+    <Card title="Image" href="/components/data-sources/image" />
     <Card title="Custom" href="/components/data-sources/custom" />
 </CardGroup>
 

docs/mint.json

@@ -84,6 +84,7 @@
                                 "components/data-sources/markdown",
                                 "components/data-sources/xml",
                                 "components/data-sources/directory",
+                                "components/data-sources/image",
                                 "components/data-sources/custom"
                             ]
                         }

examples/image/assets/test.jpg

@@ -0,0 +1,3 @@
+import baseConfig from '../../eslint.config.js';
+
+export default [...baseConfig];

examples/image/eslint.config.js

@@ -0,0 +1,11 @@
+{
+    "name": "@llm-tools/embedjs-examples-image",
+    "version": "0.1.1",
+    "type": "module",
+    "dependencies": {
+        "dotenv": "^16.4.7"
+    },
+    "scripts": {
+        "start": "nx run examples-image:serve"
+    }
+}

examples/image/package.json

@@ -0,0 +1,57 @@
+{
+    "name": "examples-image",
+    "$schema": "../../node_modules/nx/schemas/project-schema.json",
+    "sourceRoot": "examples/image/src",
+    "projectType": "application",
+    "tags": [],
+    "targets": {
+        "build": {
+            "executor": "@nx/esbuild:esbuild",
+            "outputs": ["{options.outputPath}"],
+            "defaultConfiguration": "development",
+            "options": {
+                "platform": "node",
+                "outputPath": "dist/examples/image",
+                "format": ["esm"],
+                "bundle": true,
+                "main": "examples/image/src/main.ts",
+                "tsConfig": "examples/image/tsconfig.app.json",
+                "generatePackageJson": false,
+                "esbuildOptions": {
+                    "sourcemap": true,
+                    "outExtension": {
+                        ".js": ".js"
+                    }
+                }
+            },
+            "configurations": {
+                "development": {},
+                "production": {
+                    "esbuildOptions": {
+                        "sourcemap": false,
+                        "outExtension": {
+                            ".js": ".js"
+                        }
+                    }
+                }
+            }
+        },
+        "serve": {
+            "executor": "@nx/js:node",
+            "defaultConfiguration": "development",
+            "dependsOn": ["build"],
+            "options": {
+                "buildTarget": "examples-image:build",
+                "runBuildTargetDependencies": true
+            },
+            "configurations": {
+                "development": {
+                    "buildTarget": "examples-image:build:development"
+                },
+                "production": {
+                    "buildTarget": "examples-image:build:production"
+                }
+            }
+        }
+    }
+}

examples/image/project.json

@@ -0,0 +1,17 @@
+import 'dotenv/config';
+import path from 'node:path';
+import { RAGApplicationBuilder, SIMPLE_MODELS } from '@llm-tools/embedjs';
+import { ImageLoader } from '@llm-tools/embedjs-loader-image';
+import { OpenAiEmbeddings } from '@llm-tools/embedjs-openai';
+import { HNSWDb } from '@llm-tools/embedjs-hnswlib';
+
+const ragApplication = await new RAGApplicationBuilder()
+    .setModel(SIMPLE_MODELS.OPENAI_GPT4_O)
+    .setEmbeddingModel(new OpenAiEmbeddings())
+    .setVectorDatabase(new HNSWDb())
+    .build();
+
+const imagePath = path.resolve('./examples/image/assets/test.jpg');
+await ragApplication.addLoader(new ImageLoader({ filePathOrUrl: imagePath }));
+
+await ragApplication.query('How does deep learning relate to artifical intelligence');

examples/image/src/main.ts

@@ -0,0 +1,9 @@
+{
+    "extends": "./tsconfig.json",
+    "compilerOptions": {
+        "outDir": "../../dist/out-tsc",
+        "types": ["node"]
+    },
+    "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"],
+    "include": ["src/**/*.ts"]
+}

examples/image/tsconfig.app.json

@@ -0,0 +1,17 @@
+{
+    "extends": "../../tsconfig.base.json",
+    "files": [],
+    "include": [],
+    "references": [
+        {
+            "path": "./tsconfig.app.json"
+        }
+    ],
+    "compilerOptions": {
+        "esModuleInterop": true,
+        "target": "ES2022",
+        "lib": ["ES2022", "ES2022.Object"],
+        "module": "NodeNext",
+        "moduleResolution": "nodenext"
+    }
+}