refac

Author: tjbck

backend/open_webui/config.py

@@ -1109,6 +1109,12 @@ def reachable(host: str, port: int) -> bool:
     mineru_params,
 )
 
+MINERU_FILE_EXTENSIONS = ConfigVar(
+    'MINERU_FILE_EXTENSIONS',
+    'rag.mineru_file_extensions',
+    [ext.strip() for ext in os.getenv('MINERU_FILE_EXTENSIONS', 'pdf').split(',') if ext.strip()],
+)
+
 EXTERNAL_DOCUMENT_LOADER_URL = ConfigVar(
     'EXTERNAL_DOCUMENT_LOADER_URL',
     'rag.external_document_loader_url',

backend/open_webui/main.py

@@ -281,6 +281,7 @@
     MINERU_API_MODE,
     MINERU_API_TIMEOUT,
     MINERU_API_URL,
+    MINERU_FILE_EXTENSIONS,
     MINERU_PARAMS,
     MISTRAL_OCR_API_BASE_URL,
     MISTRAL_OCR_API_KEY,
@@ -1022,6 +1023,7 @@ async def lifespan(app: FastAPI):
 app.state.config.MINERU_API_KEY = MINERU_API_KEY
 app.state.config.MINERU_API_TIMEOUT = MINERU_API_TIMEOUT
 app.state.config.MINERU_PARAMS = MINERU_PARAMS
+app.state.config.MINERU_FILE_EXTENSIONS = MINERU_FILE_EXTENSIONS
 
 app.state.config.TEXT_SPLITTER = RAG_TEXT_SPLITTER
 app.state.config.ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER = ENABLE_MARKDOWN_HEADER_TEXT_SPLITTER

backend/open_webui/retrieval/loaders/main.py

@@ -371,7 +371,7 @@ def _get_loader(self, filename: str, file_content_type: str, file_path: str):
                     azure_credential=DefaultAzureCredential(),
                     api_model=self.kwargs.get('DOCUMENT_INTELLIGENCE_MODEL'),
                 )
-        elif self.engine == 'mineru' and file_ext in ['pdf']:  # MinerU currently only supports PDF
+        elif self.engine == 'mineru' and file_ext in self.kwargs.get('MINERU_FILE_EXTENSIONS', ['pdf']):
             mineru_timeout = self.kwargs.get('MINERU_API_TIMEOUT', 300)
             if mineru_timeout:
                 try:

backend/open_webui/routers/retrieval.py

@@ -466,6 +466,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
         'MINERU_API_KEY': request.app.state.config.MINERU_API_KEY,
         'MINERU_API_TIMEOUT': request.app.state.config.MINERU_API_TIMEOUT,
         'MINERU_PARAMS': request.app.state.config.MINERU_PARAMS,
+        'MINERU_FILE_EXTENSIONS': request.app.state.config.MINERU_FILE_EXTENSIONS,
         # Reranking settings
         'RAG_RERANKING_MODEL': request.app.state.config.RAG_RERANKING_MODEL,
         'RAG_RERANKING_ENGINE': request.app.state.config.RAG_RERANKING_ENGINE,
@@ -681,6 +682,7 @@ class ConfigForm(BaseModel):
     MINERU_API_KEY: str | None = None
     MINERU_API_TIMEOUT: str | None = None
     MINERU_PARAMS: dict | None = None
+    MINERU_FILE_EXTENSIONS: list[str] | None = None
 
     # Reranking settings
     RAG_RERANKING_MODEL: str | None = None
@@ -904,6 +906,11 @@ async def update_rag_config(request: Request, form_data: ConfigForm, user=Depend
     request.app.state.config.MINERU_PARAMS = (
         form_data.MINERU_PARAMS if form_data.MINERU_PARAMS is not None else request.app.state.config.MINERU_PARAMS
     )
+    request.app.state.config.MINERU_FILE_EXTENSIONS = (
+        form_data.MINERU_FILE_EXTENSIONS
+        if form_data.MINERU_FILE_EXTENSIONS is not None
+        else request.app.state.config.MINERU_FILE_EXTENSIONS
+    )
 
     # Reranking settings
     if request.app.state.config.RAG_RERANKING_ENGINE == '':

src/lib/components/admin/Settings/Documents.svelte

@@ -244,7 +244,10 @@
 			MINERU_PARAMS:
 				typeof RAGConfig.MINERU_PARAMS === 'string' && RAGConfig.MINERU_PARAMS.trim() !== ''
 					? JSON.parse(RAGConfig.MINERU_PARAMS)
-					: {}
+					: {},
+			MINERU_FILE_EXTENSIONS: RAGConfig.MINERU_FILE_EXTENSIONS.split(',')
+				.map((ext) => ext.trim())
+				.filter((ext) => ext !== '')
 		});
 		dispatch('save');
 	};
@@ -286,6 +289,8 @@
 				? JSON.stringify(config.MINERU_PARAMS ?? {}, null, 2)
 				: config.MINERU_PARAMS;
 
+		config.MINERU_FILE_EXTENSIONS = (config?.MINERU_FILE_EXTENSIONS ?? ['pdf']).join(', ');
+
 		RAGConfig = config;
 	});
 </script>
@@ -763,6 +768,25 @@
 									/>
 								</div>
 							</div>
+
+							<!-- File Extensions -->
+							<div class="flex flex-col justify-between w-full mt-2">
+								<div class="text-xs font-medium mb-1">
+									<Tooltip
+										content={$i18n.t(
+											'Comma-separated list of file extensions MinerU will handle (e.g. pdf, docx, pptx, xlsx)'
+										)}
+										placement="top-start"
+									>
+										{$i18n.t('File Extensions')}
+									</Tooltip>
+								</div>
+								<input
+									class="flex-1 w-full text-sm bg-transparent outline-hidden"
+									placeholder={$i18n.t('pdf, docx, pptx, xlsx')}
+									bind:value={RAGConfig.MINERU_FILE_EXTENSIONS}
+								/>
+							</div>
 						{/if}
 					</div>