add encrypted db data (#11808)

Author: hamishfagg

.github/workflows/tests_integration.yml

@@ -65,4 +65,3 @@ jobs:
       env:
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         INTERNAL_URL: ${{ vars.MINDSDB_INTERNAL_URL }}
-    

mindsdb/interfaces/storage/db.py

@@ -1,6 +1,7 @@
 import json
 import orjson
 import datetime
+import os
 from typing import Dict, List, Optional
 
 import numpy as np
@@ -29,6 +30,7 @@
     sessionmaker,
 )
 from sqlalchemy.sql.schema import ForeignKey
+from mind_castle.sqlalchemy_type import SecretData
 
 from mindsdb.utilities.json_encoder import CustomJSONEncoder
 from mindsdb.utilities.config import config
@@ -137,6 +139,11 @@ def process_result_value(self, value, dialect):  # select
         return json.loads(value) if value is not None else None
 
 
+# Use MindsDB's "Json" column type as a backend for mind-castle
+class SecretDataJson(SecretData):
+    impl = Json
+
+
 class PREDICTOR_STATUS:
     __slots__ = ()
     COMPLETE = "complete"
@@ -228,7 +235,7 @@ class Integration(Base):
     created_at = Column(DateTime, default=datetime.datetime.now)
     name = Column(String, nullable=False)
     engine = Column(String, nullable=False)
-    data = Column(Json)
+    data = Column(SecretDataJson(os.environ.get("MINDSDB_DATA_ENCRYPTION_TYPE", "none")))
     company_id = Column(String)
 
     __table_args__ = (UniqueConstraint("name", "company_id", name="unique_integration_name_company_id"),)

requirements/requirements.txt

@@ -71,3 +71,5 @@ pydantic_core>=2.33.2
 
 # When using agents, some LLMs may require the 'transformers' library (like Ollama):
 transformers >= 4.42.4
+
+mind-castle >= 0.4.5