Replace codeinterpreterapi with PythonREPL and update models to gpt-5.4

The interpreter tool previously relied on codeinterpreterapi (remote sandbox) which is no longer
compatible with the current dependency stack. This replaces it with langchain_experimental
PythonREPL (local execution) while preserving the same agent contract.

Key changes:
- tool_interpreter.py: rewrite using PythonREPL — injects file previews (columns, dtypes,
  3-row sample) into the LLM prompt so generated code uses exact column names; truncates
  REPL stdout at 2000 chars to prevent context overflow; wires shared llm_instance from
  the agent instead of instantiating a separate model
- agent.py: passes llm_instance to tool constructor via import_tools introspection
- params.ini: bump default OpenAI models from gpt-4o / gpt-4o-mini to gpt-5.4 / gpt-5.4-mini
- requirements.txt / environment.yml: pin langchain_experimental==0.3.4 and remove
  codeinterpreterapi / codeboxapi dependencies
- Remove custom_sqlite_file.py (unused after langgraph checkpoint refactor)

Security note: PythonREPL executes LLM-generated code in-process without sandboxing.
A warning comment has been added; restrict to trusted deployment environments.

Author: lfnothias

app/config/params.ini

@@ -1,16 +1,16 @@
 [llm_preview]
-id = gpt-4o
+id = gpt-5.4
 temperature = 0.3
 max_retries = 3
 
 [llm_o]
-id = gpt-4o
+id = gpt-5.4
 temperature = 0
 max_retries = 3
 
 
 [llm_mini]
-id = gpt-4o-mini
+id = gpt-5.4-mini
 temperature = 0
 max_retries = 3
 
@@ -48,7 +48,7 @@ max_retries = 3
 base_url = https://llama-3-1-70b-instruct.endpoints.kepler.ai.cloud.ovh.net/api/openai_compat/v1
 
 [llm_litellm_openai]
-id = gpt-4o
+id = gpt-5.4
 temperature = 0
 
 

app/core/agents/enpkg/tool_taxon.py

@@ -5,8 +5,6 @@
 
 from pydantic import BaseModel, Field
 
-from typing import Optional
-
 from langchain.callbacks.manager import (
     CallbackManagerForToolRun,
 )

app/core/agents/interpreter/agent.py

@@ -18,6 +18,7 @@ def create_agent(llms, graph, openai_key, session_id) -> AgentExecutor:
     tool_parameters = {
         "openai_key": openai_key,
         "session_id": session_id,
+        "llm_instance": llms.get(MODEL_CHOICE),
     }
 
     tools = import_tools(directory, module_prefix, **tool_parameters)

app/core/agents/interpreter/tool_interpreter.py

@@ -1,29 +1,35 @@
 from __future__ import annotations
 
+# SECURITY WARNING: This tool uses langchain_experimental PythonREPL, which executes
+# LLM-generated Python code directly in the server process with no sandboxing.
+# Only deploy in trusted environments where the server host is not exposed to
+# untrusted users. Do not use in public-facing deployments without additional
+# isolation (e.g. Docker, restricted OS user, network policy).
 
-from codeinterpreterapi import CodeInterpreterSession, File, settings
-from pydantic import BaseModel, Field
-from langchain.tools import BaseTool
-
-from typing import Optional
-
-from langchain.callbacks.manager import (
-    CallbackManagerForToolRun,
-)
 import json
-from app.core.session import setup_logger, create_user_session
-
 import os
 import re
-import tempfile
 from pathlib import Path
+from typing import Any, Optional
+
+from langchain.callbacks.manager import CallbackManagerForToolRun
+from langchain.tools import BaseTool
+from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_experimental.utilities import PythonREPL
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel, Field
+
 from app.core.memory.database_manager import tools_database
+from app.core.session import create_user_session, setup_logger
 
 logger = setup_logger(__name__)
 
 
 class InterpreterInput(BaseModel):
-    input: str = Field(description="Input from Interpreter Agent containing the user's question, necessary file paths and other information.")
+    input: str = Field(
+        description="Input from Interpreter Agent containing the user's question, necessary file paths and other information."
+    )
+
 
 class Interpreter(BaseTool):
 
@@ -38,111 +44,135 @@ class Interpreter(BaseTool):
         None: Outputs the response after interpreting files.
     """
     args_schema: type[BaseModel] = InterpreterInput
-    openai_key: str = None
-    session_id: str = None
+    openai_key: Optional[str] = None
+    session_id: Optional[str] = None
+    llm_instance: Optional[Any] = None
 
-    def __init__(self, openai_key: str, session_id: str):
+    def __init__(self, openai_key: Optional[str] = None, session_id: Optional[str] = None, llm_instance: Optional[Any] = None):
         super().__init__()
-        self.openai_key = openai_key
+        self.openai_key = openai_key or os.getenv("OPENAI_API_KEY")
         self.session_id = session_id
+        self.llm_instance = llm_instance
 
     def _run(
         self,
         input: str,
         run_manager: Optional[CallbackManagerForToolRun] = None,
-    ) -> None:
-
+    ) -> str:
         logger.info(f"Input: {input}")
 
         file_paths = self.extract_file_paths(input)
         logger.info(f"File paths: {file_paths}")
 
         session_dir = create_user_session(self.session_id, user_session_dir=True)
-
-        settings.OPENAI_API_KEY = self.openai_key
-        settings.MODEL = "gpt-3.5-turbo"
-
-        with CodeInterpreterSession() as session:
-
-            db_manager = tools_database()
-
-            user_request = (
-                "You are an interpreter helping to analyze different questions, files and outputs generated from a series of LLMs."
-                f"The details of the current request: {input}"
-                "Please interpret the current request to generate a meaningful answer."
-                "Here's some instructions that you have to follow for acheiving the task:"
-                "1. For any file provided, analyse if and provide clear and brief information about it unless something else is asked."
-                "2. Only if a specific visualization (e.g., bar chart, diagram) is requested in the question, use the provided information to generate a .json file containing the JSON code for a Plotly graph. This file should be named identically to the analyzed file."
-                "3. After you finish your tasks, your answer should contain both the interpretation asked and the full visualization file name if visualization was requested."
-            )
-
-            files = []
-
-            for file in file_paths:
-                files.append(File.from_path(file))
-                logger.info(f"File added to interpreter Agent: {file}")
-
-            if not files:
-                
-                logger.info("No files provided from the Interpreter Agent. Manually scrapping the database")
-                output_merged = db_manager.get("tool_merge_result")  # Get all file paths from the database associated with the tool_interpreter
-                if output_merged is not None:
-                    output_merged_json = json.loads(output_merged)
-                    merged_filepaths = output_merged_json['output']['paths']
-
-                    for merged_filepath in merged_filepaths:
-                        logger.info(f"File added to interpreter Agent from the Output Merged tool: {merged_filepath}")
-                        files.append(File.from_path(merged_filepath))
-
-            if not files:
-
-                sparql_output = db_manager.get("tool_sparql")  # Get all file paths from the database associated with the tool_interpreter
-                if sparql_output is not None:
-                    sparql_output_json = json.loads(sparql_output)
-                    sparql_filepaths = sparql_output_json['output']['paths']
-
-                    for sparql_filepath in sparql_filepaths:
-                        logger.info(f"File added to interpreter Agent from the SPARQL tool: {sparql_filepath}")
-                        files.append(File.from_path(sparql_filepath))
-                
-            # generate the response
-            logger.info(f"Files submitted: {files}")
-            response = session.generate_response(user_request, files=files)
-            logger.info(f"Interpreter Agent Response: {response}")
-
-            filepaths = []
-
-            # Handling and saving output files
-            if response.files:
-                for file in response.files:
-                    logger.info(f"File: {file.name}")
-
-                    generated_file_path = session_dir / file.name
-                    with open(generated_file_path, 'wb') as f:
-                        f.write(file.content)
-                    filepaths.append(str(generated_file_path))
-
-                    logger.info(f"File saved: {file.name}")
-            else:
-                logger.info("No files generated by Interpreter Tool.")
-
-            output_data = {
-                "output": {
-                    "paths": filepaths
-                }
-            }
-
-            
+        db_manager = tools_database()
+
+        if not file_paths:
+            output_merged = db_manager.get("tool_merge_result")
+            if output_merged is not None:
+                merged_json = json.loads(output_merged)
+                file_paths = merged_json.get("output", {}).get("paths", [])
+                logger.info(f"File paths from merge result: {file_paths}")
+
+        if not file_paths:
+            sparql_output = db_manager.get("tool_sparql")
+            if sparql_output is not None:
+                sparql_json = json.loads(sparql_output)
+                file_paths = sparql_json.get("output", {}).get("paths", [])
+                logger.info(f"File paths from SPARQL tool: {file_paths}")
+
+        llm = self.llm_instance or ChatOpenAI(
+            api_key=self.openai_key,
+            model="gpt-5.4",
+            temperature=0,
+        )
+
+        system_prompt = (
+            "You are a Python data analysis assistant.\n"
+            "Write a self-contained Python script that reads the provided file(s), "
+            "fulfils the user's request, and prints a concise text summary to stdout.\n\n"
+            "Rules:\n"
+            "- pandas, numpy, json, pathlib, and plotly are available.\n"
+            "- Use the exact column names shown in the file previews — do not guess them.\n"
+            "- Print a SHORT summary (counts, stats, top-N rows) — never print an entire dataframe.\n"
+            f"- Save any output files (visualizations, processed data) to: {session_dir}\n"
+            "- For visualizations use Plotly and write JSON with: "
+            f'fig.write_json(str(Path("{session_dir}") / "<same-stem-as-input>.json"))\n'
+            "- Do NOT call plt.show(), fig.show(), or any interactive display.\n"
+            "- Do NOT install packages inside the script.\n"
+            "- Return ONLY the Python code inside a single ```python ... ``` block."
+        )
+
+        file_previews = []
+        for fp in file_paths:
             try:
-                db_manager.put(data=json.dumps(output_data), tool_name="tool_interpreter")
-            except Exception as e:
-                logger.error(f"Error saving to database: {e}")
-
-            return f"{response}.\n\n The full path of the files generated are:\n" + "\n".join(filepaths)
+                import pandas as pd
+                sep = "\t" if str(fp).endswith((".tsv", ".txt")) else ","
+                preview = pd.read_csv(fp, sep=sep, nrows=3)
+                # Truncate long cell values so wide/URL-heavy columns don't blow context
+                preview = preview.map(
+                    lambda v: str(v)[:80] + "…" if isinstance(v, str) and len(str(v)) > 80 else v
+                )
+                file_previews.append(
+                    f"File: {fp}\n"
+                    f"Columns: {preview.columns.tolist()}\n"
+                    f"Dtypes: {preview.dtypes.to_dict()}\n"
+                    f"Preview (3 rows):\n{preview.to_string(index=False)}"
+                )
+            except Exception:
+                file_previews.append(f"File: {fp} (could not preview)")
+
+        preview_block = "\n\n".join(file_previews) if file_previews else "(no files)"
+        user_message = f"File previews:\n{preview_block}\n\nRequest: {input}"
+
+        logger.info("Requesting analysis code from LLM")
+        response = llm.invoke(
+            [SystemMessage(content=system_prompt), HumanMessage(content=user_message)]
+        )
+
+        code_blocks = re.findall(r"```python\s*(.*?)\s*```", response.content, re.DOTALL)
+        if not code_blocks:
+            code_blocks = re.findall(r"```\s*(.*?)\s*```", response.content, re.DOTALL)
+
+        if not code_blocks:
+            logger.error("LLM did not return a Python code block")
+            return "Interpreter could not generate analysis code for this request."
+
+        code = code_blocks[0]
+        logger.info(f"Generated code:\n{code}")
+
+        # Snapshot existing files so we can detect what was newly created
+        existing_files = set(session_dir.iterdir()) if session_dir.exists() else set()
+
+        repl = PythonREPL()
+        execution_output = repl.run(code)
+        # Truncate to keep the agent context manageable (~2000 chars ≈ ~500 tokens)
+        _MAX_OUTPUT = 2000
+        if len(execution_output) > _MAX_OUTPUT:
+            execution_output = execution_output[:_MAX_OUTPUT] + f"\n… [output truncated at {_MAX_OUTPUT} chars]"
+        logger.info(f"Execution output: {execution_output}")
+
+        # Collect files written by the executed code
+        new_files = (
+            set(session_dir.iterdir()) - existing_files if session_dir.exists() else set()
+        )
+        filepaths = [str(f) for f in new_files]
+        logger.info(f"Files generated: {filepaths}")
+
+        output_data = {"output": {"paths": filepaths}}
+        try:
+            db_manager.put(data=json.dumps(output_data), tool_name="tool_interpreter")
+        except Exception as e:
+            logger.error(f"Error saving to database: {e}")
+
+        suffix = (
+            "\n\nThe full path of the files generated are:\n" + "\n".join(filepaths)
+            if filepaths
+            else ""
+        )
+        return f"{execution_output}{suffix}"
 
     def extract_file_paths(self, text: str):
-        # Regex to find file paths or filenames with extensions, possibly surrounded by quotes
         regex = r"['\"]?([a-zA-Z0-9_/\\-]+(?:\.csv|\.tsv|\.mgf|\.txt|\.xlsx|\.xls))['\"]?"
         matches = re.finditer(regex, text)
-        file_paths = [match.group(1).replace("'", "").replace('"', '') for match in matches]
-        return file_paths
+        return [match.group(1).replace("'", "").replace('"', "") for match in matches]