Image retrieval

backend/agents/create_agent_info.py

@@ -1,4 +1,4 @@
-import threading
+import threading
 import logging
 from typing import List, Optional
 from urllib.parse import urljoin
@@ -450,14 +450,17 @@ async def create_tool_config_list(agent_id, tenant_id, user_id, version_no: int
             rerank = param_dict.get("rerank", False)
             rerank_model_name = param_dict.get("rerank_model_name", "")
             rerank_model = None
+            is_multimodal = bool(tool_config.params.pop("multimodal", False))
             if rerank and rerank_model_name:
                 rerank_model = get_rerank_model(
                     tenant_id=tenant_id, model_name=rerank_model_name
                 )
 
             tool_config.metadata = {
                 "vdb_core": get_vector_db_core(),
-                "embedding_model": get_embedding_model(tenant_id=tenant_id),
+                "embedding_model": get_embedding_model(
+                    tenant_id=tenant_id, is_multimodal=is_multimodal
+                ),
                 "rerank_model": rerank_model,
             }
         elif tool_config.class_name in ["DifySearchTool", "DataMateSearchTool"]:

backend/apps/file_management_app.py

@@ -116,12 +116,13 @@ async def upload_files(
 
 @file_management_config_router.post("/process")
 async def process_files(
-        files: List[dict] = Body(
-            ..., description="List of file details to process, including path_or_url and filename"),
-        chunking_strategy: Optional[str] = Body("basic"),
-        index_name: str = Body(...),
-        destination: str = Body(...),
-        authorization: Optional[str] = Header(None)
+        files: Annotated[List[dict], Body(
+            ..., description="List of file details to process, including path_or_url and filename")],
+        index_name: Annotated[str, Body(...)],
+        destination: Annotated[str, Body(...)],
+        chunking_strategy: Annotated[Optional[str], Body(...)] = "basic",
+        model_id: Annotated[Optional[int], Body(...)] = None,
+        authorization: Annotated[Optional[str], Header()] = None
 ):
     """
     Trigger data processing for a list of uploaded files.
@@ -134,7 +135,8 @@ async def process_files(
         chunking_strategy=chunking_strategy,
         source_type=destination,
         index_name=index_name,
-        authorization=authorization
+        authorization=authorization,
+        model_id=model_id
     )
 
     process_result = await trigger_data_process(files, process_params)

backend/apps/model_managment_app.py

@@ -33,7 +33,7 @@
 from fastapi.responses import JSONResponse
 from fastapi.encoders import jsonable_encoder
 from http import HTTPStatus
-from typing import List, Optional
+from typing import Annotated, List, Optional
 from services.model_health_service import (
     check_model_connectivity,
     verify_model_config_connectivity,
@@ -297,7 +297,8 @@ async def get_llm_model_list(authorization: Optional[str] = Header(None)):
 
 @router.post("/healthcheck")
 async def check_model_health(
-        display_name: str = Query(..., description="Display name to check"),
+        display_name: Annotated[str, Query(..., description="Display name to check")],
+        model_type: Annotated[str, Query(..., description="...")],
         authorization: Optional[str] = Header(None)
 ):
     """Check and update model connectivity, returning the latest status.
@@ -308,7 +309,7 @@ async def check_model_health(
     """
     try:
         _, tenant_id = get_current_user_id(authorization)
-        result = await check_model_connectivity(display_name, tenant_id)
+        result = await check_model_connectivity(display_name, tenant_id, model_type)
         return JSONResponse(status_code=HTTPStatus.OK, content={
             "message": "Successfully checked model connectivity",
             "data": result

backend/apps/vectordatabase_app.py

@@ -65,11 +65,13 @@ def create_new_index(
         # Extract optional fields from request body
         ingroup_permission = None
         group_ids = None
-        embedding_model_name = None
+        is_multimodal = False
+        embedding_model_name: Optional[str] = None
         if request:
             ingroup_permission = request.get("ingroup_permission")
             group_ids = request.get("group_ids")
-            embedding_model_name = request.get("embedding_model_name")
+            is_multimodal = request.get("is_multimodal", False)
+            embedding_model_name = request.get("embeddingModel")
 
         # Treat path parameter as user-facing knowledge base name for new creations
         return ElasticSearchService.create_knowledge_base(
@@ -81,6 +83,7 @@ def create_new_index(
             ingroup_permission=ingroup_permission,
             group_ids=group_ids,
             embedding_model_name=embedding_model_name,
+            is_multimodal=is_multimodal,
         )
     except Exception as e:
         raise HTTPException(
@@ -124,13 +127,15 @@ async def update_index(
         knowledge_name = request.get("knowledge_name")
         ingroup_permission = request.get("ingroup_permission")
         group_ids = request.get("group_ids")
+        is_multimodal = request.get("is_multimodal")
 
         # Call service layer to update knowledge base
         result = ElasticSearchService.update_knowledge_base(
             index_name=index_name,
             knowledge_name=knowledge_name,
             ingroup_permission=ingroup_permission,
             group_ids=group_ids,
+            is_multimodal=is_multimodal,
             tenant_id=tenant_id,
             user_id=user_id,
         )
@@ -200,13 +205,23 @@ def create_index_documents(
         user_id, tenant_id = get_current_user_id(authorization)
 
         # Get the knowledge base record to retrieve the saved embedding model
-        knowledge_record = get_knowledge_record({'index_name': index_name})
+        knowledge_record = get_knowledge_record(
+            {"index_name": index_name, "tenant_id": tenant_id}
+        )
         saved_embedding_model_name = None
         if knowledge_record:
             saved_embedding_model_name = knowledge_record.get('embedding_model_name')
-
-        # Use the saved model from knowledge base, fallback to tenant default if not set
-        embedding_model = get_embedding_model(tenant_id, saved_embedding_model_name)
+        is_multimodal = (
+            True if knowledge_record and knowledge_record.get('is_multimodal') == 'Y' else False
+        )
+
+        # Use the saved model from knowledge base, fallback to tenant default if not set.
+        embedding_model = get_embedding_model(
+            tenant_id=tenant_id,
+            is_multimodal=is_multimodal,
+            model_name=saved_embedding_model_name,
+            strict_model_name=bool(saved_embedding_model_name),
+        )
 
         return ElasticSearchService.index_documents(
             embedding_model=embedding_model,
@@ -463,6 +478,7 @@ def update_chunk(
             chunk_request=payload,
             vdb_core=vdb_core,
             user_id=user_id,
+            tenant_id=tenant_id,
         )
         return JSONResponse(status_code=HTTPStatus.OK, content=result)
     except ValueError as e:
@@ -529,8 +545,17 @@ async def hybrid_search(
     """Run a hybrid (accurate + semantic) search across indices."""
     try:
         _, tenant_id = get_current_user_id(authorization)
+        resolved_index_names: List[str] = []
+        for requested_name in payload.index_names:
+            try:
+                resolved_name = get_index_name_by_knowledge_name(
+                    requested_name, tenant_id
+                )
+            except Exception:
+                resolved_name = requested_name
+            resolved_index_names.append(resolved_name)
         result = ElasticSearchService.search_hybrid(
-            index_names=payload.index_names,
+            index_names=resolved_index_names,
             query=payload.query,
             tenant_id=tenant_id,
             top_k=payload.top_k,

backend/consts/const.py

@@ -28,6 +28,10 @@ class VectorDatabaseType(str, Enum):
 # Data Processing Service Configuration
 DATA_PROCESS_SERVICE = os.getenv("DATA_PROCESS_SERVICE")
 CLIP_MODEL_PATH = os.getenv("CLIP_MODEL_PATH")
+TABLE_TRANSFORMER_MODEL_PATH = os.getenv("TABLE_TRANSFORMER_MODEL_PATH")
+UNSTRUCTURED_DEFAULT_MODEL_INITIALIZE_PARAMS_JSON_PATH = os.getenv(
+    "UNSTRUCTURED_DEFAULT_MODEL_INITIALIZE_PARAMS_JSON_PATH"
+)
 
 
 # Upload Configuration
@@ -115,6 +119,7 @@ class VectorDatabaseType(str, Enum):
 MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY")
 MINIO_REGION = os.getenv("MINIO_REGION")
 MINIO_DEFAULT_BUCKET = os.getenv("MINIO_DEFAULT_BUCKET")
+S3_URL_PREFIX = "s3://"
 
 
 # Postgres Configuration

backend/consts/model.py

@@ -234,6 +234,7 @@ class ProcessParams(BaseModel):
     source_type: str
     index_name: str
     authorization: Optional[str] = None
+    model_id: Optional[int] = None
 
 
 class OpinionRequest(BaseModel):