update local ov support for agent

Signed-off-by: Yongbozzz <yongbo.zhu@intel.com>

Author: Yongbozzz

EdgeCraftRAG/Dockerfile.server

@@ -18,11 +18,13 @@ RUN mkdir /templates && \
 COPY ./edgecraftrag/prompt_template/default_prompt.txt /templates/
 RUN chown -R user /templates/default_prompt.txt
 
-COPY ./edgecraftrag /home/user/edgecraftrag
-
-RUN mkdir -p /home/user/ui_cache
+RUN mkdir -p /home/user/ui_cache /home/user/edgecraftrag
 ENV UI_UPLOAD_PATH=/home/user/ui_cache
 
+# Copy requirements first so pip install is cached independently from source changes
+COPY ./edgecraftrag/requirements.txt /home/user/edgecraftrag/requirements.txt
+RUN chown -R user /home/user/edgecraftrag
+
 USER user
 
 WORKDIR /home/user/edgecraftrag
@@ -37,4 +39,7 @@ ENV PYTHONPATH="$PYTHONPATH:/home/user/genai/tools/llm_bench"
 
 RUN python3 -m nltk.downloader -d /home/user/nltk_data punkt_tab averaged_perceptron_tagger_eng
 
+# Copy the full source last — changes here no longer bust the pip cache layers above
+COPY ./edgecraftrag /home/user/edgecraftrag
+
 ENTRYPOINT ["python3", "-m", "edgecraftrag.server"]

EdgeCraftRAG/edgecraftrag/components/generator.py

@@ -393,16 +393,51 @@ def __init__(self, llm_model, inference_type, vllm_endpoint, **kwargs):
                 self.model_path = llm_instance.model_path
 
         self.llm = llm_model
+        if self.inference_type == InferenceType.LOCAL:
+            self.lock = asyncio.Lock()
         if self.inference_type == InferenceType.VLLM:
             self.vllm_name = llm_model().model_id
             if vllm_endpoint == "":
                 vllm_endpoint = os.getenv("vLLM_ENDPOINT", "http://localhost:8086")
         self.vllm_endpoint = vllm_endpoint
 
     async def run(self, chat_request, retrieved_nodes, node_parser_type, **kwargs):
-        response = await self.run_vllm(chat_request, retrieved_nodes, node_parser_type, **kwargs)
+        if self.inference_type == InferenceType.LOCAL:
+            response = await self.run_local(chat_request, retrieved_nodes, node_parser_type, **kwargs)
+        elif self.inference_type == InferenceType.VLLM:
+            response = await self.run_vllm(chat_request, retrieved_nodes, node_parser_type, **kwargs)
+        else:
+            raise ValueError("LLM inference_type not supported")
         return response
 
+    async def run_local(self, chat_request, retrieved_nodes, node_parser_type, **kwargs):
+        if self.llm() is None:
+            # This could happen when User delete all LLMs through RESTful API
+            raise ValueError("No LLM available, please load LLM")
+        generate_kwargs = dict(
+            temperature=chat_request.temperature,
+            do_sample=chat_request.temperature > 0.0,
+            top_p=chat_request.top_p,
+            top_k=chat_request.top_k,
+            typical_p=chat_request.typical_p,
+            repetition_penalty=chat_request.repetition_penalty,
+        )
+        self.llm().generate_kwargs = generate_kwargs
+        self.llm().max_new_tokens = chat_request.max_tokens
+        prompt_str = chatcompletion_to_chatml(chat_request)
+        if chat_request.stream:
+
+            # Asynchronous generator
+            async def generator():
+                async for chunk in local_stream_generator(self.lock, self.llm(), prompt_str, ""):
+                    yield chunk or ""
+                    await asyncio.sleep(0)
+
+            return generator()
+        else:
+            result = self.llm().complete(prompt_str)
+            return result
+
     async def run_vllm(self, chat_request, retrieved_nodes, node_parser_type, **kwargs):
         llm = OpenAILike(
             api_key="fake",