VLM integration via Ollama added (#339)

* VLM integration via Ollama added

Author: pashidlos

.env

@@ -37,3 +37,4 @@ AWS_S3_BUCKET_NAME=
 #SERVER_TIMEOUT=120000
 #SERVER_HEADERS_TIMEOUT=60000
 #SERVER_KEEP_ALIVE_TIMEOUT=5000
+OLLAMA_BASE_URL=http://localhost:11434

.github/workflows/workflow.yml

@@ -33,6 +33,9 @@ jobs:
       - name: Run acceptance tests
         run: npm run test:acceptance
 
+      - name: Wait for services to be ready with migrations
+        run: sleep 30
+
       - name: Run e2e tests
         run: npm run test:e2e
 

docker-compose.yml

@@ -16,10 +16,13 @@ services:
       APP_FRONTEND_URL: ${APP_FRONTEND_URL}
       BODY_PARSER_JSON_LIMIT: ${BODY_PARSER_JSON_LIMIT}
       ELASTIC_URL: ${ELASTIC_URL}
+      OLLAMA_BASE_URL: ${OLLAMA_BASE_URL}
     ports:
       - "${APP_PORT}:3000"
     expose:
       - "${APP_PORT}"
+    extra_hosts:
+      - host.docker.internal:host-gateway
     depends_on:
       postgres:
         condition: service_healthy

package-lock.json

@@ -52,6 +52,7 @@
     "ldapts": "^7.1.0",
     "looks-same": "^9.0.0",
     "odiff-bin": "^2.6.1",
+    "ollama": "^0.6.3",
     "passport": "^0.6.0",
     "passport-jwt": "^4.0.1",
     "passport-local": "^1.0.0",
@@ -63,7 +64,8 @@
     "rimraf": "^5.0.1",
     "rxjs": "^7.8.2",
     "swagger-ui-express": "^4.6.3",
-    "uuid-apikey": "^1.5.3"
+    "uuid-apikey": "^1.5.3",
+    "zod": "^4.2.1"
   },
   "devDependencies": {
     "@darraghor/eslint-plugin-nestjs-typed": "^6.9.3",

package.json

@@ -12,6 +12,4 @@ RUN npm ci --verbose
 RUN chmod +x /app/wait-for-it.sh
 RUN chmod +x /app/entrypoint.sh
 
-ENTRYPOINT ["/app/entrypoint.sh"]
-
-CMD ["sh"]
+ENTRYPOINT ["/app/entrypoint.sh"]

prisma/Dockerfile

@@ -0,0 +1,6 @@
+-- AlterEnum
+ALTER TYPE "ImageComparison" ADD VALUE 'vlm';
+
+-- AlterTable
+ALTER TABLE "TestRun" ADD COLUMN "vlmDescription" TEXT;
+

prisma/migrations/20251209181002_add_vlm_support/migration.sql

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
-provider = "postgresql"
+# It should be added in your version-control system (e.g., Git)
+provider = "postgresql"

prisma/migrations/migration_lock.toml

@@ -69,6 +69,7 @@ model TestRun {
   baselineBranchName    String?
   ignoreAreas           String         @default("[]")
   tempIgnoreAreas       String         @default("[]")
+  vlmDescription        String?
   baseline              Baseline?
   build                 Build          @relation(fields: [buildId], references: [id])
   project               Project?       @relation(fields: [projectId], references: [id])
@@ -138,6 +139,7 @@ enum ImageComparison {
   pixelmatch
   lookSame
   odiff
+  vlm
 }
 
 enum Role {

prisma/schema.prisma

@@ -96,6 +96,7 @@ export const generateTestRun = (testRun?: Partial<TestRun>): TestRun => {
     baselineBranchName: 'master',
     branchName: 'develop',
     merge: false,
+    vlmDescription: null,
     ...testRun,
   };
 };