Merge main into feature/extra-options-passthrough

Resolve conflicts in mssql and mysql adapters by combining:
- extra_options passthrough from this branch
- autocommit enablement (mssql) and charset auto-sync (mysql) from main

Author: Maxteabag

infra/docker/docker-compose.test.yml

@@ -51,6 +51,48 @@ services:
     tmpfs:
       - /var/lib/mysql
 
+  # MySQL with TIS-620 charset (Thai) - for charset testing
+  mysql-tis620:
+    image: mysql:8.0
+    container_name: sqlit-test-mysql-tis620
+    command: --character-set-server=tis620 --collation-server=tis620_thai_ci
+    environment:
+      MYSQL_ROOT_PASSWORD: "TestPassword123!"
+      MYSQL_USER: "testuser"
+      MYSQL_PASSWORD: "TestPassword123!"
+      MYSQL_DATABASE: "test_sqlit"
+    ports:
+      - "${MYSQL_TIS620_PORT:-3308}:3306"
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "testuser", "-pTestPassword123!"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    tmpfs:
+      - /var/lib/mysql
+
+  # MySQL with Latin1 charset - for charset testing
+  mysql-latin1:
+    image: mysql:8.0
+    container_name: sqlit-test-mysql-latin1
+    command: --character-set-server=latin1 --collation-server=latin1_swedish_ci
+    environment:
+      MYSQL_ROOT_PASSWORD: "TestPassword123!"
+      MYSQL_USER: "testuser"
+      MYSQL_PASSWORD: "TestPassword123!"
+      MYSQL_DATABASE: "test_sqlit"
+    ports:
+      - "${MYSQL_LATIN1_PORT:-3309}:3306"
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "testuser", "-pTestPassword123!"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    tmpfs:
+      - /var/lib/mysql
+
   clickhouse:
     image: clickhouse/clickhouse-server:latest
     container_name: sqlit-test-clickhouse

sqlit/domains/connections/providers/mariadb/adapter.py

@@ -102,6 +102,11 @@ def connect(self, config: ConnectionConfig) -> Any:
 
         connect_args.update(config.extra_options)
         conn = mariadb_any.connect(**connect_args)
+
+        # Note: The MariaDB Python connector only supports UTF-8 family charsets.
+        # Legacy charsets like TIS-620 or Latin1 are not supported. For databases
+        # using legacy charsets, use the MySQL provider with PyMySQL instead.
+
         self._supports_sequences = self._detect_sequences_support(conn)
         return conn
 

sqlit/domains/connections/providers/mssql/adapter.py

@@ -183,7 +183,10 @@ def connect(self, config: ConnectionConfig) -> Any:
         # Append extra_options to connection string
         for key, value in config.extra_options.items():
             conn_str += f"{key}={value};"
-        return mssql_python.connect(conn_str)
+        conn = mssql_python.connect(conn_str)
+        # Enable autocommit to allow DDL statements like CREATE DATABASE
+        conn.autocommit = True
+        return conn
 
     def get_databases(self, conn: Any) -> list[str]:
         """Get list of databases from SQL Server."""

sqlit/domains/connections/providers/mysql/adapter.py

@@ -111,4 +111,24 @@ def connect(self, config: ConnectionConfig) -> Any:
             connect_args["ssl"] = ssl_params
 
         connect_args.update(config.extra_options)
-        return pymysql.connect(**connect_args)
+        conn = pymysql.connect(**connect_args)
+
+        # Auto-sync charset with server to handle legacy encodings (e.g., TIS-620, Latin1).
+        # This ensures data is read correctly when the database uses a non-UTF-8 charset.
+        try:
+            cursor = conn.cursor()
+            cursor.execute("SELECT @@character_set_database")
+            row = cursor.fetchone()
+            if row:
+                server_charset = row[0]
+                # Only switch if server uses a different charset than our default (utf8mb4)
+                if server_charset and server_charset.lower() != "utf8mb4":
+                    # Use set_charset() which both sends SET NAMES AND updates
+                    # PyMySQL's internal encoding for proper byte decoding
+                    conn.set_charset(server_charset)
+            cursor.close()
+        except Exception:
+            # If charset sync fails, continue with default - better than failing completely
+            pass
+
+        return conn

sqlit/domains/connections/providers/oracle/adapter.py

@@ -72,8 +72,18 @@ def connect(self, config: ConnectionConfig) -> Any:
         if endpoint is None:
             raise ValueError("Oracle connections require a TCP-style endpoint.")
         port = int(endpoint.port or get_default_port("oracle"))
-        # Use Easy Connect string format: host:port/service_name
-        dsn = f"{endpoint.host}:{port}/{endpoint.database}"
+
+        # Determine connection type: service_name (default) or sid
+        connection_type = config.get_option("oracle_connection_type", "service_name")
+
+        if connection_type == "sid":
+            # SID format: host:port:sid (uses colon separator)
+            # SID is stored in oracle_sid field, fall back to database for backward compat
+            sid = config.get_option("oracle_sid") or endpoint.database
+            dsn = f"{endpoint.host}:{port}:{sid}"
+        else:
+            # Service Name format: host:port/service_name (uses slash separator)
+            dsn = f"{endpoint.host}:{port}/{endpoint.database}"
 
         # Determine connection mode based on oracle_role
         oracle_role = config.get_option("oracle_role", "normal")

sqlit/domains/connections/providers/oracle/schema.py

@@ -20,6 +20,21 @@ def _get_oracle_role_options() -> tuple[SelectOption, ...]:
     )
 
 
+def _get_oracle_connection_type_options() -> tuple[SelectOption, ...]:
+    return (
+        SelectOption("service_name", "Service Name"),
+        SelectOption("sid", "SID"),
+    )
+
+
+def _oracle_connection_type_is_service_name(values: dict) -> bool:
+    return values.get("oracle_connection_type", "service_name") != "sid"
+
+
+def _oracle_connection_type_is_sid(values: dict) -> bool:
+    return values.get("oracle_connection_type") == "sid"
+
+
 SCHEMA = ConnectionSchema(
     db_type="oracle",
     display_name="Oracle",
@@ -32,11 +47,26 @@ def _get_oracle_role_options() -> tuple[SelectOption, ...]:
             group="server_port",
         ),
         _port_field("1521"),
+        SchemaField(
+            name="oracle_connection_type",
+            label="Connection Type",
+            field_type=FieldType.DROPDOWN,
+            options=_get_oracle_connection_type_options(),
+            default="service_name",
+        ),
         SchemaField(
             name="database",
             label="Service Name",
             placeholder="ORCL or XEPDB1",
             required=True,
+            visible_when=_oracle_connection_type_is_service_name,
+        ),
+        SchemaField(
+            name="oracle_sid",
+            label="SID",
+            placeholder="ORCL",
+            required=True,
+            visible_when=_oracle_connection_type_is_sid,
         ),
         _username_field(),
         _password_field(),

sqlit/domains/connections/providers/oracle_legacy/schema.py

@@ -20,6 +20,21 @@ def _get_oracle_role_options() -> tuple[SelectOption, ...]:
     )
 
 
+def _get_oracle_connection_type_options() -> tuple[SelectOption, ...]:
+    return (
+        SelectOption("service_name", "Service Name"),
+        SelectOption("sid", "SID"),
+    )
+
+
+def _oracle_connection_type_is_service_name(values: dict) -> bool:
+    return values.get("oracle_connection_type", "service_name") != "sid"
+
+
+def _oracle_connection_type_is_sid(values: dict) -> bool:
+    return values.get("oracle_connection_type") == "sid"
+
+
 def _get_oracle_client_mode_options() -> tuple[SelectOption, ...]:
     return (
         SelectOption("thick", "Thick (Instant Client)"),
@@ -43,11 +58,26 @@ def _oracle_thick_mode_enabled(values: dict) -> bool:
             group="server_port",
         ),
         _port_field("1521"),
+        SchemaField(
+            name="oracle_connection_type",
+            label="Connection Type",
+            field_type=FieldType.DROPDOWN,
+            options=_get_oracle_connection_type_options(),
+            default="service_name",
+        ),
         SchemaField(
             name="database",
             label="Service Name",
+            placeholder="ORCL or XEPDB1",
+            required=True,
+            visible_when=_oracle_connection_type_is_service_name,
+        ),
+        SchemaField(
+            name="oracle_sid",
+            label="SID",
             placeholder="ORCL",
             required=True,
+            visible_when=_oracle_connection_type_is_sid,
         ),
         _username_field(),
         _password_field(),

sqlit/domains/explorer/app/schema_service.py

@@ -94,9 +94,11 @@ def list_folder_items(self, folder_type: str, database: str | None) -> list[Any]
         cache_key = database or "__default__"
         obj_cache = self.object_cache
 
-        def cached(key: str, loader: Callable[[], Any]) -> Any:
+        def cached(key: str, loader: Callable[[], Any], *, allow_empty: bool = True) -> Any:
             if cache_key in obj_cache and key in obj_cache[cache_key]:
-                return obj_cache[cache_key][key]
+                data = obj_cache[cache_key][key]
+                if allow_empty or data:
+                    return data
             data = loader()
             if cache_key not in obj_cache:
                 obj_cache[cache_key] = {}
@@ -110,6 +112,7 @@ def cached(key: str, loader: Callable[[], Any]) -> Any:
                     lambda: inspector.get_tables(self.session.connection, db_arg),
                     database,
                 ),
+                allow_empty=self.session.provider.metadata.db_type != "duckdb",
             )
             return [("table", schema, name) for schema, name in raw_data]
         if folder_type == "views":
@@ -119,6 +122,7 @@ def cached(key: str, loader: Callable[[], Any]) -> Any:
                     lambda: inspector.get_views(self.session.connection, db_arg),
                     database,
                 ),
+                allow_empty=self.session.provider.metadata.db_type != "duckdb",
             )
             return [("view", schema, name) for schema, name in raw_data]
         if folder_type == "databases":

sqlit/domains/explorer/ui/mixins/tree.py

@@ -210,9 +210,21 @@ def on_tree_node_highlighted(self: TreeMixinHost, event: Tree.NodeHighlighted) -
 
     def action_refresh_tree(self: TreeMixinHost) -> None:
         """Refresh the explorer."""
+        self._refresh_tree_common(notify=True)
+
+    def _refresh_tree_after_schema_change(self: TreeMixinHost) -> None:
+        """Refresh tree after DDL without showing a notification."""
+        self._refresh_tree_common(notify=False)
+
+    def _refresh_tree_common(self: TreeMixinHost, *, notify: bool) -> None:
         self._get_object_cache().clear()
-        if hasattr(self, "_schema_cache") and "columns" in self._schema_cache:
+        if hasattr(self, "_schema_cache") and isinstance(self._schema_cache, dict):
             self._schema_cache["columns"] = {}
+            self._schema_cache["tables"] = []
+            self._schema_cache["views"] = []
+            self._schema_cache["procedures"] = []
+        if hasattr(self, "_db_object_cache"):
+            self._db_object_cache = {}
         if hasattr(self, "_loading_nodes"):
             self._loading_nodes.clear()
         self._schema_service = None
@@ -253,7 +265,8 @@ def run_loader() -> None:
                 )
             else:
                 self._schedule_timer(MIN_TIMER_DELAY_S, run_loader)
-        self.notify("Refreshed")
+        if notify:
+            self.notify("Refreshed")
 
     def refresh_tree(self: TreeMixinHost) -> None:
         tree_builder.refresh_tree_chunked(self)

sqlit/domains/query/ui/mixins/query_execution.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import re
+
 from typing import TYPE_CHECKING, Any, Callable
 
 from sqlit.domains.explorer.ui.tree import db_switching as tree_db_switching
@@ -21,6 +23,19 @@
     from sqlit.domains.query.app.transaction import TransactionExecutor
 
 
+_SCHEMA_CHANGE_RE = re.compile(
+    r"\b(create|alter|drop|truncate|rename|comment|grant|revoke)\b",
+    re.IGNORECASE,
+)
+_SQL_COMMENT_RE = re.compile(r"(--[^\n]*|/\*.*?\*/)", re.DOTALL)
+_SQL_LITERAL_RE = re.compile(r"('([^']|'')*'|\"([^\"]|\"\")*\"|`[^`]*`|\[[^\]]*\])", re.DOTALL)
+
+
+def _strip_sql_comments_and_literals(sql: str) -> str:
+    sql = _SQL_COMMENT_RE.sub(" ", sql)
+    return _SQL_LITERAL_RE.sub(" ", sql)
+
+
 class QueryExecutionMixin(ProcessWorkerLifecycleMixin):
     """Mixin providing query execution actions."""
 
@@ -216,6 +231,21 @@ def _on_result(confirmed: bool | None) -> None:
             _on_result,
         )
 
+    def _query_changes_schema(self: QueryMixinHost, query: str) -> bool:
+        cleaned = _strip_sql_comments_and_literals(query)
+        return bool(_SCHEMA_CHANGE_RE.search(cleaned))
+
+    def _maybe_refresh_explorer_after_query(self: QueryMixinHost, query: str) -> None:
+        if not self._query_changes_schema(query):
+            return
+        refresh = getattr(self, "_refresh_tree_after_schema_change", None)
+        if callable(refresh):
+            refresh()
+            return
+        action = getattr(self, "action_refresh_tree", None)
+        if callable(action):
+            action()
+
     def _start_query_spinner(self: QueryMixinHost) -> None:
         """Start the query execution spinner animation."""
         import time
@@ -481,6 +511,7 @@ async def _run_query_async(self: QueryMixinHost, query: str, keep_insert_mode: b
                         )
                     else:
                         self._display_non_query_result(result.rows_affected, elapsed_ms)
+                    self._maybe_refresh_explorer_after_query(query)
                     if keep_insert_mode:
                         self._restore_insert_mode()
                     return
@@ -500,6 +531,7 @@ async def _run_query_async(self: QueryMixinHost, query: str, keep_insert_mode: b
                 except Exception:
                     pass
                 self._display_multi_statement_results(multi_result, elapsed_ms)
+                self._maybe_refresh_explorer_after_query(query)
             else:
                 # Single statement - existing behavior
                 result = await asyncio.to_thread(
@@ -520,6 +552,7 @@ async def _run_query_async(self: QueryMixinHost, query: str, keep_insert_mode: b
                     )
                 else:
                     self._display_non_query_result(result.rows_affected, elapsed_ms)
+                self._maybe_refresh_explorer_after_query(query)
 
             if keep_insert_mode:
                 self._restore_insert_mode()
@@ -584,14 +617,17 @@ async def _run_query_atomic_async(self: QueryMixinHost, query: str) -> None:
                     self.notify("Transaction rolled back (error in statement)", severity="error")
                 else:
                     self.notify("Query executed atomically (committed)", severity="information")
+                    self._maybe_refresh_explorer_after_query(query)
             elif isinstance(result, QueryResult):
                 await self._display_query_results(
                     result.columns, result.rows, result.row_count, result.truncated, elapsed_ms
                 )
                 self.notify("Query executed atomically (committed)", severity="information")
+                self._maybe_refresh_explorer_after_query(query)
             else:
                 self._display_non_query_result(result.rows_affected, elapsed_ms)
                 self.notify("Query executed atomically (committed)", severity="information")
+                self._maybe_refresh_explorer_after_query(query)
 
         except Exception as e:
             self._display_query_error(f"Transaction rolled back: {e}")