VLM integration via Ollama added

Author: pashidlos

.env

@@ -37,3 +37,4 @@ AWS_S3_BUCKET_NAME=
 #SERVER_TIMEOUT=120000
 #SERVER_HEADERS_TIMEOUT=60000
 #SERVER_KEEP_ALIVE_TIMEOUT=5000
+OLLAMA_BASE_URL=http://localhost:11434

docker-compose.yml

@@ -16,10 +16,15 @@ services:
       APP_FRONTEND_URL: ${APP_FRONTEND_URL}
       BODY_PARSER_JSON_LIMIT: ${BODY_PARSER_JSON_LIMIT}
       ELASTIC_URL: ${ELASTIC_URL}
+      # VLM: Uncomment to use Ollama running on host machine
+      OLLAMA_BASE_URL: http://host.docker.internal:11434
     ports:
       - "${APP_PORT}:3000"
     expose:
       - "${APP_PORT}"
+    # VLM: Uncomment to use Ollama running on host machine
+    extra_hosts:
+      - host.docker.internal:host-gateway
     depends_on:
       postgres:
         condition: service_healthy

prisma/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
-provider = "postgresql"
+# It should be added in your version-control system (e.g., Git)
+provider = "postgresql"

prisma/schema.prisma

@@ -69,6 +69,7 @@ model TestRun {
   baselineBranchName    String?
   ignoreAreas           String         @default("[]")
   tempIgnoreAreas       String         @default("[]")
+  vlmDescription        String?
   baseline              Baseline?
   build                 Build          @relation(fields: [buildId], references: [id])
   project               Project?       @relation(fields: [projectId], references: [id])
@@ -138,6 +139,7 @@ enum ImageComparison {
   pixelmatch
   lookSame
   odiff
+  vlm
 }
 
 enum Role {

src/_data_/index.ts

@@ -96,6 +96,7 @@ export const generateTestRun = (testRun?: Partial<TestRun>): TestRun => {
     baselineBranchName: 'master',
     branchName: 'develop',
     merge: false,
+    vlmDescription: null,
     ...testRun,
   };
 };

src/compare/compare.module.ts

@@ -3,10 +3,14 @@ import { CompareService } from './compare.service';
 import { LookSameService } from './libs/looks-same/looks-same.service';
 import { OdiffService } from './libs/odiff/odiff.service';
 import { PixelmatchService } from './libs/pixelmatch/pixelmatch.service';
+import { VlmService } from './libs/vlm/vlm.service';
+import { OllamaController } from './libs/vlm/ollama.controller';
+import { OllamaService } from './libs/vlm/ollama.service';
 import { StaticModule } from '../static/static.module';
 
 @Module({
-  providers: [CompareService, PixelmatchService, LookSameService, OdiffService],
+  controllers: [OllamaController],
+  providers: [CompareService, PixelmatchService, LookSameService, OdiffService, VlmService, OllamaService],
   imports: [StaticModule],
   exports: [CompareService],
 })

src/compare/compare.service.ts

@@ -7,6 +7,7 @@ import { PrismaService } from '../prisma/prisma.service';
 import { DiffResult } from '../test-runs/diffResult';
 import { LookSameService } from './libs/looks-same/looks-same.service';
 import { OdiffService } from './libs/odiff/odiff.service';
+import { VlmService } from './libs/vlm/vlm.service';
 import { isHddStaticServiceConfigured } from '../static/utils';
 
 @Injectable()
@@ -17,6 +18,7 @@ export class CompareService {
     private readonly pixelmatchService: PixelmatchService,
     private readonly lookSameService: LookSameService,
     private readonly odiffService: OdiffService,
+    private readonly vlmService: VlmService,
     private readonly prismaService: PrismaService
   ) {}
 
@@ -44,6 +46,9 @@ export class CompareService {
 
         return this.odiffService;
       }
+      case ImageComparison.vlm: {
+        return this.vlmService;
+      }
       default: {
         this.logger.warn(`Unknown ImageComparison value: ${imageComparison}. Falling back to pixelmatch.`);
         return this.pixelmatchService;

src/compare/libs/image-comparator.interface.ts

@@ -3,8 +3,9 @@ import { ImageCompareInput } from './ImageCompareInput';
 import { LooksSameConfig } from './looks-same/looks-same.types';
 import { OdiffConfig } from './odiff/odiff.types';
 import { PixelmatchConfig } from './pixelmatch/pixelmatch.types';
+import { VlmConfig } from './vlm/vlm.types';
 
-export type ImageCompareConfig = PixelmatchConfig | LooksSameConfig | OdiffConfig;
+export type ImageCompareConfig = PixelmatchConfig | LooksSameConfig | OdiffConfig | VlmConfig;
 
 export interface ImageComparator {
   getDiff(data: ImageCompareInput, config: ImageCompareConfig): Promise<DiffResult>;

src/compare/libs/vlm/README.md

@@ -0,0 +1,92 @@
+# VLM (Vision Language Model) Image Comparison
+
+AI-powered semantic image comparison using Vision Language Models via Ollama.
+
+## Quick Start
+
+### 1. Install & Start Ollama
+
+```bash
+# Install (macOS)
+brew install ollama
+
+# Start Ollama
+ollama serve
+```
+
+### 2. Download a Model
+
+```bash
+# Recommended for accuracy
+ollama pull llava:7b
+
+# Or for speed (smaller, less accurate)
+ollama pull moondream
+```
+
+### 3. Configure Backend
+
+Add to `.env`:
+```bash
+OLLAMA_BASE_URL=http://localhost:11434
+```
+
+### 4. Use VLM in Project
+
+Set project's image comparison to `vlm` with config:
+```json
+{
+  "model": "llava:7b",
+  "temperature": 0.1
+}
+```
+
+Optional custom prompt:
+```json
+{
+  "model": "llava:7b",
+  "prompt": "Focus on button colors and text changes",
+  "temperature": 0.1
+}
+```
+
+## Recommended Models
+
+| Model | Size | Speed | Accuracy | Best For |
+|-------|------|-------|----------|----------|
+| `llava:7b` | 4.7GB | ⚡⚡ | ⭐⭐⭐ | **Recommended** - best balance |
+| `llava:13b` | 8GB | ⚡ | ⭐⭐⭐⭐ | Best accuracy |
+| `moondream` | 1.7GB | ⚡⚡⚡ | ⭐⭐ | Fast, may hallucinate |
+| `minicpm-v` | 5.5GB | ⚡⚡ | ⭐⭐⭐ | Good alternative |
+
+## Configuration
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `model` | string | `moondream` | Ollama vision model name |
+| `prompt` | string | `""` | Custom context prepended to system prompt |
+| `temperature` | number | `0.1` | Lower = more consistent results |
+
+## How It Works
+
+1. VLM analyzes both images semantically
+2. Returns `YES` (pass) or `NO` (fail) based on meaningful differences
+3. Ignores technical differences (anti-aliasing, sub-pixel, minor spacing)
+4. Provides description of differences found
+
+## API Endpoints
+
+```bash
+# List available models
+GET /ollama/models
+
+# Compare two images (for testing)
+POST /ollama/compare?model=llava:7b&prompt=<prompt>&temperature=0.1
+```
+
+**Example:**
+```bash
+curl -X POST "http://localhost:3000/ollama/compare?model=llava:7b&prompt=Are%20these%20images%20the%20same&temperature=0.1" \
+  -F "images=@baseline.png" \
+  -F "images=@comparison.png"
+```

src/compare/libs/vlm/ollama.controller.ts

@@ -0,0 +1,50 @@
+import { Controller, Get, Post, Query, HttpException, HttpStatus, UseInterceptors, UploadedFiles } from '@nestjs/common';
+import { FilesInterceptor } from '@nestjs/platform-express';
+import { ApiTags, ApiConsumes, ApiBody } from '@nestjs/swagger';
+import { OllamaService } from './ollama.service';
+
+@ApiTags('Ollama')
+@Controller('ollama')
+export class OllamaController {
+  constructor(private readonly ollamaService: OllamaService) {}
+
+  @Get('models')
+  async listModels() {
+    return { models: await this.ollamaService.listModels() };
+  }
+
+  @Post('compare')
+  @ApiConsumes('multipart/form-data')
+  @ApiBody({
+    schema: {
+      type: 'object',
+      required: ['images'],
+      properties: {
+        images: {
+          type: 'array',
+          items: { type: 'string', format: 'binary' },
+          description: 'Two images to compare (baseline and comparison)',
+        },
+      },
+    },
+  })
+  @UseInterceptors(FilesInterceptor('images', 2))
+  async compareImages(
+    @UploadedFiles() files: Express.Multer.File[],
+    @Query('model') model: string,
+    @Query('prompt') prompt: string,
+    @Query('temperature') temperature: string
+  ) {
+    if (files?.length !== 2) {
+      throw new HttpException('Two images required', HttpStatus.BAD_REQUEST);
+    }
+
+    return this.ollamaService.generate({
+      model,
+      prompt,
+      format: 'json',
+      images: files.map((f) => f.buffer.toString('base64')),
+      options: { temperature: Number(temperature) },
+    });
+  }
+}