perf(optimizer): speed up qualify by ~24% and optimize by ~17% (#7724)

- Add Expression.meta_get(key, default), a non-allocating meta read: the
  meta property allocates {} on first read, so hot per-node read paths
  (normalize_identifiers' prune, simplify's FINAL checks) were allocating
  a dict on every AST node and slowing every subsequent copy(). Convert
  all read-only .meta.get(...) sites; writes still go through meta.
- Scope._collect: a single tuple-isinstance gate (COLLECTIBLE_TYPES) lets
  non-collectible nodes skip the classification chain, paid 3x per query
  (qualify_tables, qualify_columns, validate_qualify_columns).
- walk_in_scope: only CTE/Query nodes can start child scopes, so one
  isinstance gate replaces up to 4 checks + 2 parent loads per node.
- Scope.local_columns: identity-based set instead of structural-equality
  set of column nodes.
- parse_identifier: skip the tokenizer/parser round-trip for names
  matching SAFE_IDENTIFIER_RE (provably identical output).
- _expand_using: early-exit for scopes with no joins / no USING joins,
  preserving the missing-source validation.
- quote_identifiers: direct walk over Identifier nodes instead of
  transform(), which only does replacement bookkeeping here.

Qualified SQL is byte-identical on all TPC-H/TPC-DS fixture queries.
Pure Python and mypyc release builds both pass the full suite.

Author: tobymao

sqlglot/dialects/bigquery.py

@@ -147,9 +147,9 @@ def normalize_identifier(self, expression: E) -> E:
                 or (
                     isinstance(parent, exp.Table)
                     and parent.db
-                    and (parent.meta.get("quoted_table") or not parent.meta.get("maybe_column"))
+                    and (parent.meta_get("quoted_table") or not parent.meta_get("maybe_column"))
                 )
-                or expression.meta.get("is_table")
+                or expression.meta_get("is_table")
             )
             if not case_sensitive:
                 expression.set("this", expression.this.lower())

sqlglot/expressions/builders.py

@@ -319,6 +319,11 @@ def parse_identifier(name: str | Identifier, dialect: DialectType = None) -> Ide
     Returns:
         The identifier ast node.
     """
+    if isinstance(name, str) and SAFE_IDENTIFIER_RE.match(name):
+        # Simple names parse to a single unquoted identifier in all dialects, so we can
+        # avoid the tokenizer/parser round-trip for them.
+        return Identifier(this=name, quoted=False)
+
     try:
         expression = maybe_parse(name, dialect=dialect, into=Identifier)
     except (ParseError, TokenError):
@@ -822,7 +827,7 @@ def replace_tables(
     mapping = {normalize_table_name(k, dialect=dialect): v for k, v in mapping.items()}
 
     def _replace_tables(node: Expr) -> Expr:
-        if isinstance(node, Table) and node.meta.get("replace") is not False:
+        if isinstance(node, Table) and node.meta_get("replace") is not False:
             original = normalize_table_name(node, dialect=dialect)
             new_name = mapping.get(original)
 

sqlglot/expressions/core.py

@@ -240,6 +240,9 @@ def is_leaf(self) -> bool:
     def meta(self) -> dict[str, t.Any]:
         raise NotImplementedError
 
+    def meta_get(self, key: str, default: t.Any = None) -> t.Any:
+        raise NotImplementedError
+
     def __deepcopy__(self, memo: t.Any) -> Expr:
         raise NotImplementedError
 
@@ -990,6 +993,11 @@ def meta(self) -> dict[str, t.Any]:
             self._meta = {}
         return self._meta
 
+    def meta_get(self, key: str, default: t.Any = None) -> t.Any:
+        """Reads a meta value without allocating the meta dict (unlike the `meta` property)."""
+        meta = self._meta
+        return meta.get(key, default) if meta is not None else default
+
     def __deepcopy__(self, memo: t.Any) -> Expr:
         root = self.__class__()
         stack: list[tuple[Expr, Expr]] = [(self, root)]

sqlglot/generator.py

@@ -4554,7 +4554,7 @@ def function_fallback_sql(self, expression: exp.Func) -> str:
                 args.append(arg_value)
 
         if self.dialect.PRESERVE_ORIGINAL_NAMES:
-            name = (expression._meta and expression.meta.get("name")) or expression.sql_name()
+            name = expression.meta_get("name") or expression.sql_name()
         else:
             name = expression.sql_name()
 
@@ -5258,7 +5258,7 @@ def _embed_ignore_nulls(self, expression: exp.IgnoreNulls | exp.RespectNulls, te
             )
             return self.sql(this)
 
-        if self.IGNORE_NULLS_IN_FUNC and not expression.meta.get("inline"):
+        if self.IGNORE_NULLS_IN_FUNC and not expression.meta_get("inline"):
             if self.IGNORE_NULLS_BEFORE_ORDER:
                 # The first modifier here will be the one closest to the AggFunc's arg
                 mods = sorted(

sqlglot/generators/bigquery.py

@@ -201,7 +201,7 @@ def _levenshtein_sql(self: BigQueryGenerator, expression: exp.Levenshtein) -> st
 
 
 def _json_extract_sql(self: BigQueryGenerator, expression: JSON_EXTRACT_TYPE) -> str:
-    name = (expression._meta and expression.meta.get("name")) or expression.sql_name()
+    name = expression.meta_get("name") or expression.sql_name()
     upper = name.upper()
 
     dquote_escaping = upper in DQUOTES_ESCAPING_JSON_FUNCTIONS
@@ -565,7 +565,7 @@ def mod_sql(self, expression: exp.Mod) -> str:
         )
 
     def column_parts(self, expression: exp.Column) -> str:
-        if expression.meta.get("quoted_column"):
+        if expression.meta_get("quoted_column"):
             # If a column reference is of the form `dataset.table`.name, we need
             # to preserve the quoted table path, otherwise the reference breaks
             table_parts = ".".join(p.name for p in expression.parts[:-1])
@@ -583,7 +583,7 @@ def table_parts(self, expression: exp.Table) -> str:
         #
         # - WITH x AS (SELECT [1, 2] AS y) SELECT * FROM x, `x.y`   -> cross join
         # - WITH x AS (SELECT [1, 2] AS y) SELECT * FROM x, `x`.`y` -> implicit unnest
-        if expression.meta.get("quoted_table"):
+        if expression.meta_get("quoted_table"):
             table_parts = ".".join(p.name for p in expression.parts)
             return self.sql(exp.Identifier(this=table_parts, quoted=True))
 

sqlglot/optimizer/annotate_types.py

@@ -400,7 +400,7 @@ def annotate_scope(self, scope: Scope) -> None:
                     isinstance(source, Scope)
                     and isinstance(source.expression, exp.Query)
                     and (
-                        source.expression.meta.get("query_type") or exp.DType.UNKNOWN.into_expr()
+                        source.expression.meta_get("query_type") or exp.DType.UNKNOWN.into_expr()
                     ).is_type(exp.DType.STRUCT)
                 ):
                     self._set_type(table_column, source.expression.meta["query_type"])
@@ -497,7 +497,7 @@ def _annotate_expression(
                 else:
                     self._set_type(expr, exp.DType.UNKNOWN)
 
-                if expr.is_type(exp.DType.JSON) and (dot_parts := expr.meta.get("dot_parts")):
+                if expr.is_type(exp.DType.JSON) and (dot_parts := expr.meta_get("dot_parts")):
                     # JSON dot access is case sensitive across all dialects, so we need to undo the normalization.
                     i = iter(dot_parts)
                     parent = expr.parent
@@ -740,7 +740,7 @@ def _annotate_binary(self, expression: B) -> B:
             self._annotate_by_args(expression, left, right)
 
         if isinstance(expression, exp.Is) or (
-            left.meta.get("nonnull") is True and right.meta.get("nonnull") is True
+            left.meta_get("nonnull") is True and right.meta_get("nonnull") is True
         ):
             expression.meta["nonnull"] = True
 
@@ -752,7 +752,7 @@ def _annotate_unary(self, expression: E) -> E:
         else:
             self._set_type(expression, expression.this.type)
 
-        if expression.this.meta.get("nonnull") is True:
+        if expression.this.meta_get("nonnull") is True:
             expression.meta["nonnull"] = True
 
         return expression

sqlglot/optimizer/normalize_identifiers.py

@@ -63,8 +63,8 @@ def normalize_identifiers(expression, dialect=None, store_original_column_identi
     if isinstance(expression, str):
         expression = exp.parse_identifier(expression, dialect=dialect)
 
-    for node in expression.walk(prune=lambda n: bool(n.meta.get("case_sensitive"))):
-        if not node.meta.get("case_sensitive"):
+    for node in expression.walk(prune=lambda n: bool(n.meta_get("case_sensitive"))):
+        if not node.meta_get("case_sensitive"):
             if store_original_column_identifiers and isinstance(node, exp.Column):
                 # TODO: This does not handle non-column cases, e.g PARSE_JSON(...).key
                 parent = node
@@ -73,6 +73,7 @@ def normalize_identifiers(expression, dialect=None, store_original_column_identi
 
                 node.meta["dot_parts"] = [p.name for p in parent.parts]
 
-            dialect.normalize_identifier(node)
+            if isinstance(node, exp.Identifier):
+                dialect.normalize_identifier(node)
 
     return expression

sqlglot/optimizer/qualify_columns.py

@@ -124,10 +124,10 @@ def validate_qualify_columns(expression: E, sql: str | None = None) -> E:
             if scope.external_columns and not scope.is_correlated_subquery and not scope.pivots:
                 column = scope.external_columns[0]
                 for_table = f" for table: '{column.table}'" if column.table else ""
-                line = column.this.meta.get("line")
-                col = column.this.meta.get("col")
-                start = column.this.meta.get("start")
-                end = column.this.meta.get("end")
+                line = column.this.meta_get("line")
+                col = column.this.meta_get("col")
+                start = column.this.meta_get("start")
+                end = column.this.meta_get("end")
 
                 error_msg = f"Column '{column.name}' could not be resolved{for_table}."
                 if line and col:
@@ -142,10 +142,10 @@ def validate_qualify_columns(expression: E, sql: str | None = None) -> E:
 
     if all_unqualified_columns:
         first_column = all_unqualified_columns[0]
-        line = first_column.this.meta.get("line")
-        col = first_column.this.meta.get("col")
-        start = first_column.this.meta.get("start")
-        end = first_column.this.meta.get("end")
+        line = first_column.this.meta_get("line")
+        col = first_column.this.meta_get("col")
+        start = first_column.this.meta_get("start")
+        end = first_column.this.meta_get("end")
 
         error_msg = f"Ambiguous column '{first_column.name}'"
         if line and col:
@@ -204,6 +204,9 @@ def _update_source_columns(source_name: str) -> None:
                 columns[column_name] = source_name
 
     joins = list(scope.find_all(exp.Join))
+    if not joins:
+        return {}
+
     names = {join.alias_or_name for join in joins}
     ordered = [key for key in scope.selected_sources if key not in names]
 
@@ -213,6 +216,9 @@ def _update_source_columns(source_name: str) -> None:
     # Mapping of automatically joined column names to an ordered set of source names (dict).
     column_tables: dict[str, dict[str, t.Any]] = {}
 
+    if not any(join.args.get("using") for join in joins):
+        return column_tables
+
     for source_name in ordered:
         _update_source_columns(source_name)
 
@@ -954,9 +960,15 @@ def qualify_outputs(scope_or_expression: Scope | exp.Expr) -> None:
 
 def quote_identifiers(expression: E, dialect: DialectType = None, identify: bool = True) -> E:
     """Makes sure all identifiers that need to be quoted are quoted."""
-    return expression.transform(
-        Dialect.get_or_raise(dialect).quote_identifier, identify=identify, copy=False
-    )
+    dialect = Dialect.get_or_raise(dialect)
+
+    # `quote_identifier` only mutates identifiers in place, so we avoid `transform` here
+    # because its node replacement machinery is wasteful for this case.
+    for node in expression.walk():
+        if isinstance(node, exp.Identifier):
+            dialect.quote_identifier(node, identify=identify)
+
+    return expression
 
 
 def pushdown_cte_alias_columns(scope: Scope) -> None:

sqlglot/optimizer/scope.py

@@ -21,6 +21,20 @@
 
 ROW_LEVEL_AGG_FUNCS = (exp.Count,)
 
+# The node types `Scope._collect` classifies, roughly ordered by frequency. `exp.Query`
+# covers subqueries, and `exp.UDTF` covers laterals.
+COLLECTIBLE_TYPES = (
+    exp.Column,
+    exp.Dot,
+    exp.Table,
+    exp.Query,
+    exp.UDTF,
+    exp.CTE,
+    exp.Star,
+    exp.TableColumn,
+    exp.JoinHint,
+)
+
 
 class ScopeType(Enum):
     ROOT = auto()
@@ -170,7 +184,9 @@ def _collect(self) -> None:
         self._column_index = set()
 
         for node in self.walk():
-            if node is self.expression:
+            # Most nodes (identifiers, literals, operators etc.) aren't collectible, so a
+            # single isinstance gate lets them skip the classification chain below.
+            if node is self.expression or not isinstance(node, COLLECTIBLE_TYPES):
                 continue
 
             if isinstance(node, exp.Dot) and node.is_star:
@@ -460,8 +476,10 @@ def local_columns(self) -> list[exp.Column]:
             list[exp.Column]: Column instances that reference sources in the current scope.
         """
         if self._local_columns is None:
-            external_columns = set(self.external_columns)
-            self._local_columns = [c for c in self.columns if c not in external_columns]
+            # Compare nodes by identity: structural equality would conflate distinct
+            # column nodes that happen to look the same, and is much more expensive.
+            external_column_ids = {id(c) for c in self.external_columns}
+            self._local_columns = [c for c in self.columns if id(c) not in external_column_ids]
 
         return self._local_columns
 
@@ -950,11 +968,17 @@ def walk_in_scope(
 
         yield node
 
-        if node is not expression and (
-            isinstance(node, exp.CTE)
-            or (isinstance(node.parent, (exp.From, exp.Join)) and _is_derived_table(node))
-            or (isinstance(node.parent, exp.UDTF) and isinstance(node, exp.Query))
-            or isinstance(node, exp.UNWRAPPED_QUERIES)
+        # Only CTEs and Queries can start child scopes; checking that first lets all
+        # other nodes (the vast majority) skip the rest of the boundary checks.
+        if (
+            node is not expression
+            and isinstance(node, (exp.CTE, exp.Query))
+            and (
+                isinstance(node, exp.CTE)
+                or (isinstance(node.parent, (exp.From, exp.Join)) and _is_derived_table(node))
+                or isinstance(node.parent, exp.UDTF)
+                or isinstance(node, exp.UNWRAPPED_QUERIES)
+            )
         ):
             if isinstance(node, (exp.Subquery, exp.UDTF)):
                 for key in ("joins", "laterals", "pivots"):

sqlglot/optimizer/simplify.py

@@ -596,9 +596,9 @@ def simplify(
         joins: list[exp.Join] = []
 
         for node in expression.walk(
-            prune=lambda n: bool(isinstance(n, exp.Condition) or n.meta.get(FINAL))
+            prune=lambda n: bool(isinstance(n, exp.Condition) or n.meta_get(FINAL))
         ):
-            if node.meta.get(FINAL):
+            if node.meta_get(FINAL):
                 continue
 
             # group by expressions cannot be simplified, for example
@@ -687,10 +687,10 @@ def _simplify(
                 original.replace(node)
 
             for n in node.iter_expressions(reverse=True):
-                if n.meta.get(FINAL):
+                if n.meta_get(FINAL):
                     raise
             pre_transformation_stack.extend(
-                n for n in node.iter_expressions(reverse=True) if not n.meta.get(FINAL)
+                n for n in node.iter_expressions(reverse=True) if not n.meta_get(FINAL)
             )
             post_transformation_stack.append((node, parent))
 
@@ -937,7 +937,7 @@ def remove_complements(self, expression: object, root: bool = True) -> object:
             ops = set(expression.flatten())
             for op in ops:
                 if isinstance(op, exp.Not) and op.this in ops:
-                    if expression.meta.get("nonnull") is True:
+                    if expression.meta_get("nonnull") is True:
                         return exp.false() if isinstance(expression, exp.And) else exp.true()
 
         return expression