Handle SQLParseError gracefully by disabling grouping

Author: kkm-horikawa

debug_toolbar/panels/sql/utils.py

@@ -5,6 +5,7 @@
 from django.dispatch import receiver
 from django.test.signals import setting_changed
 from sqlparse import tokens as T
+from sqlparse.exceptions import SQLParseError
 
 from debug_toolbar import settings as dt_settings
 
@@ -91,42 +92,7 @@ def is_select_query(sql):
     return sql.lower().lstrip(" (").startswith("select")
 
 
-def _format_skipped_sql(sql, reason):
-    """
-    Format SQL that was skipped from prettification.
-
-    Shows a notice and the first portion of raw SQL for debugging.
-    """
-    preview_length = 500
-    preview = escape(sql[:preview_length])
-    total_length = len(sql)
-
-    if total_length > preview_length:
-        suffix = f"\n\n... ({total_length - preview_length:,} more characters)"
-    else:
-        suffix = ""
-
-    return (
-        f'<span class="djdt-sql-skipped">'
-        f"<em>SQL formatting skipped ({reason})</em>"
-        f"</span>"
-        f"<pre>{preview}{suffix}</pre>"
-    )
-
-
 def reformat_sql(sql, *, with_toggle=False):
-    # Check length threshold to prevent slow formatting of large queries
-    max_length = dt_settings.get_config()["SQL_PRETTIFY_MAX_LENGTH"]
-    if max_length and len(sql) > max_length:
-        skipped = _format_skipped_sql(
-            sql,
-            f"query length {len(sql):,} exceeds threshold {max_length:,}",
-        )
-        if with_toggle:
-            # Return as non-collapsible since both versions would be the same
-            return f'<span class="djDebugUncollapsed">{skipped}</span>'
-        return skipped
-
     formatted = parse_sql(sql)
     if not with_toggle:
         return formatted
@@ -140,7 +106,15 @@ def reformat_sql(sql, *, with_toggle=False):
 @lru_cache(maxsize=128)
 def parse_sql(sql, *, simplify=False):
     stack = get_filter_stack(simplify=simplify)
-    return "".join(stack.run(sql))
+    try:
+        return "".join(stack.run(sql))
+    except SQLParseError:
+        # The query either exceeds the number of tokens or depth of tokens.
+        # Recreate the FilterStack and explicitly disable the grouping to avoid
+        # those errors.
+        stack = get_filter_stack(simplify=simplify)
+        stack._grouping = False
+        return "".join(stack.run(sql))
 
 
 @cache

debug_toolbar/settings.py

@@ -50,7 +50,6 @@ def _is_running_tests():
         "django.utils.functional",
     ),
     "PRETTIFY_SQL": True,
-    "SQL_PRETTIFY_MAX_LENGTH": 50000,  # Skip formatting for SQL longer than this
     "PROFILER_CAPTURE_PROJECT_CODE": True,
     "PROFILER_MAX_DEPTH": 10,
     "PROFILER_THRESHOLD_RATIO": 8,

docs/changes.rst

@@ -4,10 +4,10 @@ Change log
 Pending
 -------
 
-* Added ``SQL_PRETTIFY_MAX_LENGTH`` setting to skip SQL formatting for
-  queries exceeding the threshold. This prevents the SQL panel from freezing
-  or crashing when queries contain large IN clauses with thousands of
-  parameters.
+* Added graceful degradation for SQL queries that exceed sqlparse's token
+  limits. When ``SQLParseError`` is raised, the SQL panel now automatically
+  disables grouping and retries formatting, preventing crashes with large
+  queries.
 * Deprecated ``RedirectsPanel`` in favor of ``HistoryPanel`` for viewing
   toolbar data from redirected requests.
 * Fixed support for generating code coverage comments in PRs.

tests/panels/test_sql.py

@@ -869,49 +869,49 @@ def test_explain_with_union(self):
         query = self.panel._queries[0]
         self.assertTrue(query["is_select"])
 
-    @override_settings(
-        DEBUG_TOOLBAR_CONFIG={"SQL_PRETTIFY_MAX_LENGTH": 100, "PRETTIFY_SQL": True}
-    )
-    def test_sql_prettify_max_length(self):
+    @override_settings(DEBUG_TOOLBAR_CONFIG={"PRETTIFY_SQL": True})
+    def test_sql_parse_error_graceful_degradation(self):
         """
-        Test that SQL formatting is skipped for queries exceeding the max length threshold.
+        Test that SQLParseError is handled gracefully by disabling grouping.
         """
-        from debug_toolbar.panels.sql.utils import reformat_sql
+        from unittest.mock import patch
 
-        # Short SQL should be formatted normally
-        short_sql = "SELECT * FROM auth_user WHERE id = 1"
-        result = reformat_sql(short_sql, with_toggle=True)
-        self.assertNotIn("SQL formatting skipped", result)
-        self.assertIn("SELECT", result)
+        from sqlparse.exceptions import SQLParseError
 
-        # Long SQL should skip formatting and show a message
-        long_sql = (
-            "SELECT * FROM auth_user WHERE id IN ("
-            + ", ".join([f"'{i}'" for i in range(100)])
-            + ")"
-        )
-        result = reformat_sql(long_sql, with_toggle=True)
-        self.assertIn("SQL formatting skipped", result)
-        self.assertIn("exceeds threshold", result)
+        from debug_toolbar.panels.sql.utils import parse_sql
 
-    def test_sql_prettify_max_length_disabled(self):
-        """
-        Test that SQL_PRETTIFY_MAX_LENGTH=0 or None disables the length check.
-        """
-        from debug_toolbar.panels.sql.utils import reformat_sql
+        # Clear the cache to ensure our mock is used
+        parse_sql.cache_clear()
 
-        long_sql = (
-            "SELECT * FROM auth_user WHERE id IN ("
-            + ", ".join([f"'{i}'" for i in range(100)])
-            + ")"
-        )
+        # Mock the filter stack to raise SQLParseError on first call
+        call_count = 0
+        original_run = None
+
+        def mock_run(sql):
+            nonlocal call_count, original_run
+            call_count += 1
+            if call_count == 1:
+                raise SQLParseError("Token limit exceeded")
+            # On retry, return a simple result
+            return [sql]
+
+        with patch(
+            "debug_toolbar.panels.sql.utils.get_filter_stack"
+        ) as mock_get_stack:
+            mock_stack = mock_get_stack.return_value
+            mock_stack.run = mock_run
+            mock_stack._grouping = True
+
+            result = parse_sql("SELECT * FROM test")
+
+            # Should have been called twice (once for error, once for retry)
+            self.assertEqual(call_count, 2)
+            # On retry, _grouping should be set to False
+            self.assertFalse(mock_stack._grouping)
+            self.assertIn("SELECT", result)
 
-        with override_settings(
-            DEBUG_TOOLBAR_CONFIG={"SQL_PRETTIFY_MAX_LENGTH": 0, "PRETTIFY_SQL": True}
-        ):
-            result = reformat_sql(long_sql, with_toggle=True)
-            # When max_length is 0 (falsy), formatting should not be skipped
-            self.assertNotIn("SQL formatting skipped", result)
+        # Clear the cache after the test
+        parse_sql.cache_clear()
 
 
 class SQLPanelMultiDBTestCase(BaseMultiDBTestCase):