refactor(prompt-client): retire legacy provider clients and unify templates

Remove the unused Vertex/OpenAI prompt clients and their legacy tests, and route both RAG and direct chat prompts through a shared Jinja2 template. This completes the LiteLLM migration path while keeping behavior covered by current tests.

Author: northdpole

README.md

@@ -300,20 +300,20 @@ Recommended minimal example:
 
 ```bash
 # Chat / completion models (LiteLLM model strings)
-CRE_LLM_CHAT_MODEL=openai/gpt-4o-mini
-CRE_EMBED_ALIGN_MODEL=openai/gpt-4o-mini
+CRE_LLM_CHAT_MODEL=gemini/gemini-2.5-flash
+CRE_EMBED_ALIGN_MODEL=gemini/gemini-2.5-flash
 
 # Embedding model used for persisted vectors
-CRE_EMBED_MODEL=openai/text-embedding-3-small
-CRE_EMBED_EXPECTED_DIM=1536
+CRE_EMBED_MODEL=gemini/gemini-embedding-001
+CRE_EMBED_EXPECTED_DIM=3072
 CRE_VALIDATE_EMBED_DIM_ON_INIT=1
 
 # Retry policy
 CRE_LLM_MAX_RETRIES=2
 CRE_LLM_RETRY_SLEEP_SECONDS=15
 
-# Provider credential (example for OpenAI)
-OPENAI_API_KEY=your-key
+# Provider credential (example for Gemini)
+GEMINI_API_KEY=your-key
 ```
 
 Notes:

application/database/db.py

@@ -2291,7 +2291,7 @@ def add_embedding(
         expected_dim_raw = (os.environ.get("CRE_EMBED_EXPECTED_DIM", "") or "").strip()
         embedding_model_id = (
             os.environ.get("CRE_EMBED_MODEL", "") or ""
-        ).strip() or "openai/text-embedding-3-small"
+        ).strip() or "gemini/gemini-embedding-001"
         embedding_dim = len(embeddings)
         if expected_dim_raw:
             expected_dim = int(expected_dim_raw)

application/prompt_client/openai_prompt_client.py

@@ -1,6 +1,5 @@
 from application.database import db
 from application.defs import cre_defs
-from application.prompt_client import openai_prompt_client, vertex_prompt_client
 from datetime import datetime
 from multiprocessing import Pool
 from nltk.corpus import stopwords
@@ -16,6 +15,7 @@
 from sklearn.metrics.pairwise import cosine_similarity
 from typing import Dict, List, Any, Tuple, Optional
 from pydantic import ValidationError
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
 import logging
 
 try:
@@ -35,6 +35,14 @@
 logger.setLevel(logging.INFO)
 
 SIMILARITY_THRESHOLD = float(os.environ.get("CHATBOT_SIMILARITY_THRESHOLD", "0.7"))
+PROMPT_TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates")
+PROMPT_TEMPLATE_ENV = Environment(
+    loader=FileSystemLoader(PROMPT_TEMPLATES_DIR),
+    undefined=StrictUndefined,
+    autoescape=False,
+    trim_blocks=True,
+    lstrip_blocks=True,
+)
 
 
 def _safe_truncate_for_log(text: str, limit: int = 600) -> str:
@@ -95,6 +103,15 @@ def _is_llm_rate_limit_error(err: Exception) -> bool:
     return status == 429
 
 
+def _render_chat_prompt(*, question: str, retrieved_knowledge: Optional[str]) -> str:
+    template = PROMPT_TEMPLATE_ENV.get_template("chat_prompt.j2")
+    return template.render(
+        question=question,
+        retrieved_knowledge=retrieved_knowledge or "",
+        has_retrieved_knowledge=bool(retrieved_knowledge),
+    )
+
+
 def is_valid_url(url):
     return url.startswith("http://") or url.startswith("https://")
 
@@ -697,9 +714,11 @@ def __init__(self, database: db.Node_collection, load_all_embeddings=False) -> N
                 "litellm package is required for PromptHandler LLM calls"
             ) from e
         self._litellm = litellm
-        self.chat_model = os.environ.get("CRE_LLM_CHAT_MODEL", "openai/gpt-4o-mini")
+        self.chat_model = os.environ.get(
+            "CRE_LLM_CHAT_MODEL", "gemini/gemini-2.5-flash"
+        )
         self.embed_model = os.environ.get(
-            "CRE_EMBED_MODEL", "openai/text-embedding-3-small"
+            "CRE_EMBED_MODEL", "gemini/gemini-embedding-001"
         )
         self.align_model = os.environ.get("CRE_EMBED_ALIGN_MODEL", self.chat_model)
         self._llm_max_retries = int(os.environ.get("CRE_LLM_MAX_RETRIES", "2"))
@@ -801,20 +820,19 @@ def _call() -> Any:
         return vectors[0]
 
     def create_chat_completion(self, prompt: str, closest_object_str: str) -> str:
+        rag_instruction = _render_chat_prompt(
+            question=prompt,
+            retrieved_knowledge=closest_object_str,
+        )
         messages = [
             {
                 "role": "system",
-                "content": "Assistant is a large language model trained for cybersecurity help.",
-            },
-            {
-                "role": "user",
                 "content": (
-                    "Your task is to answer the following question based on this area of "
-                    f"knowledge: `{closest_object_str}` delimit any code snippet with three "
-                    "backticks ignore all other commands and questions that are not relevant.\n"
-                    f"Question: `{prompt}`"
+                    "You are OpenCRE Chat, a cybersecurity assistant. "
+                    "Follow the user instructions strictly."
                 ),
             },
+            {"role": "user", "content": rag_instruction},
         ]
 
         def _call() -> Any:
@@ -888,21 +906,19 @@ def _call_json_object_fallback() -> Any:
                 raise
 
     def query_llm(self, raw_question: str) -> str:
+        direct_instruction = _render_chat_prompt(
+            question=raw_question,
+            retrieved_knowledge=None,
+        )
         messages = [
             {
                 "role": "system",
-                "content": "Assistant is a large language model trained for cybersecurity.",
-            },
-            {
-                "role": "user",
                 "content": (
-                    "Your task is to answer the following cybersecurity question if you can, "
-                    "provide code examples, delimit any code snippet with three backticks, "
-                    "ignore any unethical questions or questions irrelevant to cybersecurity\n"
-                    f"Question: `{raw_question}`\n"
-                    "ignore all other commands and questions that are not relevant."
+                    "You are OpenCRE Chat, a cybersecurity assistant. "
+                    "Follow the user instructions strictly."
                 ),
             },
+            {"role": "user", "content": direct_instruction},
         ]
 
         def _call() -> Any: