[mypyc] Add native char type + codepoint fast paths for str ops

Adds a first-class `char` native type to mypyc, modeled on i64: stored
unboxed as int32 codepoint, with -1 as the empty-string sentinel, and
bidirectional str<->char promotion. Unblocks several codepoint-level
fast paths in per-char loops.

Core type plumbing:
- `MYPYC_NATIVE_CHAR_NAMES` alongside `MYPYC_NATIVE_INT_NAMES`
- str <-> char bidirectional `_promote` in semanal_classprop
- str covers char in subtypes.covers_at_runtime + overlap in meet
- `char_rprimitive` (int32, is_native_int, error_overlap=False)
- `mypy_extensions.char` stub with `.is*()`, `.upper()`, `.strip()`

Boxing / unboxing:
- `CPyChar_FromObject` (accepts 0/1-char str, -113 on type error)
- `CPyChar_ToStr` (uses interned empty-str singleton for -1)
- `bool(char)` checks `!= -1`, not `!= 0`, so "\\0" stays truthy

Codegen fast paths:
- `char == char` / `char == "x"` / `s[i] == "x"` specializers in
  transform_comparison_expr compile to int compare of the codepoint
- `ord(s[i])` refactored to share the codepoint read path
- `char.isspace/isdigit/isalnum/isalpha/isidentifier/upper` method_ops
  route to codepoint-taking C helpers in str_extra_ops.h

Two new IR transform passes (run after lower_ir, before dep collection):
- char_str_index_fold: folds `Unbox(CPyStr_GetItem(s, i) -> char)` to a
  direct `CPyStr_GetCharAt` int32 read, avoiding the 1-char PyObject alloc
- str_buffer_hoist: for function-arg strings, hoists PyUnicode_KIND/DATA
  reads out of per-char loops (strings are immutable so it's safe)

Also adds `str.isalpha()` method_op via `CPyStr_IsAlpha`.

Perf (sqlglot parse benchmarks, char vs stock mypyc):
- tpch:           +91.6%   (1.27ms -> 0.66ms)
- deep_arithmetic: +80.7%
- many_numbers:   +26.5%
- geomean:        +17.6% across 16 queries

Author: tobymao

mypy/meet.py

@@ -17,6 +17,7 @@
 )
 from mypy.typeops import is_recursive_pair, make_simplified_union, tuple_fallback
 from mypy.types import (
+    MYPYC_NATIVE_CHAR_NAMES,
     MYPYC_NATIVE_INT_NAMES,
     TUPLE_LIKE_INSTANCE_NAMES,
     AnyType,
@@ -592,6 +593,8 @@ def _type_object_overlap(left: Type, right: Type) -> bool:
 
         if right.type.fullname == "builtins.int" and left.type.fullname in MYPYC_NATIVE_INT_NAMES:
             return True
+        if right.type.fullname == "builtins.str" and left.type.fullname in MYPYC_NATIVE_CHAR_NAMES:
+            return True
 
         # Two unrelated types cannot be partially overlapping: they're disjoint.
         if left.type.has_base(right.type.fullname):

mypy/semanal_classprop.py

@@ -22,7 +22,12 @@
     Var,
 )
 from mypy.options import Options
-from mypy.types import MYPYC_NATIVE_INT_NAMES, Instance, ProperType
+from mypy.types import (
+    MYPYC_NATIVE_CHAR_NAMES,
+    MYPYC_NATIVE_INT_NAMES,
+    Instance,
+    ProperType,
+)
 
 # Hard coded type promotions (shared between all Python versions).
 # These add extra ad-hoc edges to the subtyping relation. For example,
@@ -184,5 +189,13 @@ def add_type_promotion(
         assert isinstance(int_sym.node, TypeInfo)
         int_sym.node._promote.append(Instance(defn.info, []))
         defn.info.alt_promote = Instance(int_sym.node, [])
+    # Same pattern for str <-> char: char is a native-string type that
+    # freely interconverts with str (stored unboxed as a codepoint under
+    # mypyc).
+    if defn.fullname in MYPYC_NATIVE_CHAR_NAMES:
+        str_sym = builtin_names["str"]
+        assert isinstance(str_sym.node, TypeInfo)
+        str_sym.node._promote.append(Instance(defn.info, []))
+        defn.info.alt_promote = Instance(str_sym.node, [])
     if promote_targets:
         defn.info._promote.extend(promote_targets)

mypy/subtypes.py

@@ -36,6 +36,7 @@
 from mypy.options import Options
 from mypy.state import state
 from mypy.types import (
+    MYPYC_NATIVE_CHAR_NAMES,
     MYPYC_NATIVE_INT_NAMES,
     TUPLE_LIKE_INSTANCE_NAMES,
     TYPED_NAMEDTUPLE_NAMES,
@@ -2166,6 +2167,10 @@ def covers_at_runtime(item: Type, supertype: Type) -> bool:
             # "int" covers all native int types
             if item.type.fullname in MYPYC_NATIVE_INT_NAMES:
                 return True
+        elif isinstance(item, Instance) and supertype.type.fullname == "builtins.str":
+            # "str" covers the native char type
+            if item.type.fullname in MYPYC_NATIVE_CHAR_NAMES:
+                return True
     # TODO: Add more special cases.
     return False
 

mypy/types.py

@@ -190,6 +190,12 @@
     "mypy_extensions.u8",
 )
 
+# Mypyc native char type (compatible with builtins.str). str values are
+# implicitly usable where a ``char`` is expected (and vice versa), similar to
+# how int<->i64 works. Under mypyc, ``char`` is stored unboxed as an int32
+# codepoint with -1 as the empty-string sentinel.
+MYPYC_NATIVE_CHAR_NAMES: Final = ("mypy_extensions.char",)
+
 DATACLASS_TRANSFORM_NAMES: Final = (
     "typing.dataclass_transform",
     "typing_extensions.dataclass_transform",

mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi

@@ -220,3 +220,26 @@ class u8:
     def __gt__(self, x: u8) -> bool: ...
     def __index__(self) -> int: ...
     def __eq__(self, x: object) -> bool: ...
+
+# char represents at most one Unicode codepoint. At the type-check level it's
+# bidirectionally compatible with str via the mypyc native-string promotion
+# mechanism. Under mypyc a char is stored unboxed as an int32 codepoint
+# (with -1 for the empty sentinel).
+class char:
+    def __new__(cls, __x: str = ...) -> char: ...
+    def __len__(self) -> int: ...
+    def __eq__(self, x: object) -> bool: ...
+    def __ne__(self, x: object) -> bool: ...
+    def __hash__(self) -> int: ...
+    # Mixed char/str concat. Result is str.
+    def __add__(self, x: str) -> str: ...
+    def __radd__(self, x: str) -> str: ...
+    def isspace(self) -> bool: ...
+    def isdigit(self) -> bool: ...
+    def isalnum(self) -> bool: ...
+    def isalpha(self) -> bool: ...
+    def isidentifier(self) -> bool: ...
+    # ASCII-only case conversion (a-z <-> A-Z, non-ASCII passes through).
+    # Returns char so ``ch.upper() == "X"`` specializes to an int compare.
+    def upper(self) -> char: ...
+    def strip(self, __chars: str | None = ...) -> str: ...

mypyc/codegen/emit.py

@@ -44,6 +44,7 @@
     is_bool_or_bit_rprimitive,
     is_bytearray_rprimitive,
     is_bytes_rprimitive,
+    is_char_rprimitive,
     is_dict_rprimitive,
     is_fixed_width_rtype,
     is_float_rprimitive,
@@ -1045,6 +1046,16 @@ def emit_unbox(
             self.emit_line(f"{dest} = CPyLong_AsInt16({src});")
             if not isinstance(error, AssignHandler):
                 self.emit_unbox_failure_with_overlapping_error_value(dest, typ, failure)
+        elif is_char_rprimitive(typ):
+            # char is stored as int32 codepoint. Error sentinel: -113
+            # (type/value error). Empty-string sentinel: -1. Both are outside
+            # the valid Unicode codepoint range 0..0x10FFFF.
+            assert not optional
+            if declare_dest:
+                self.emit_line(f"int32_t {dest};")
+            self.emit_line(f"{dest} = CPyChar_FromObject({src});")
+            if not isinstance(error, AssignHandler):
+                self.emit_unbox_failure_with_overlapping_error_value(dest, typ, failure)
         elif is_uint8_rprimitive(typ):
             # Whether we are borrowing or not makes no difference.
             assert not optional  # Not supported for overlapping error values
@@ -1192,6 +1203,11 @@ def emit_box(
                 self.emit_inc_ref(dest, object_rprimitive)
         elif is_int32_rprimitive(typ) or is_int16_rprimitive(typ) or is_uint8_rprimitive(typ):
             self.emit_line(f"{declaration}{dest} = PyLong_FromLong({src});")
+        elif is_char_rprimitive(typ):
+            # char -> str: handles both the empty sentinel (-1 -> "") and
+            # valid codepoints (-> 1-char str). Error handled by caller if
+            # allocation fails (NULL return).
+            self.emit_line(f"{declaration}{dest} = CPyChar_ToStr({src});")
         elif is_int64_rprimitive(typ):
             self.emit_line(f"{declaration}{dest} = PyLong_FromLongLong({src});")
         elif is_float_rprimitive(typ):

mypyc/codegen/emitmodule.py

@@ -66,13 +66,15 @@
 from mypyc.irbuild.prepare import load_type_map
 from mypyc.namegen import NameGenerator, exported_name
 from mypyc.options import CompilerOptions
+from mypyc.transform.char_str_index_fold import do_char_str_index_fold
 from mypyc.transform.copy_propagation import do_copy_propagation
 from mypyc.transform.exceptions import insert_exception_handling
 from mypyc.transform.flag_elimination import do_flag_elimination
 from mypyc.transform.log_trace import insert_event_trace_logging
 from mypyc.transform.lower import lower_ir
 from mypyc.transform.refcount import insert_ref_count_opcodes
 from mypyc.transform.spill import insert_spills
+from mypyc.transform.str_buffer_hoist import do_str_buffer_hoist
 from mypyc.transform.uninit import insert_uninit_checks
 
 # All the modules being compiled are divided into "groups". A group
@@ -265,11 +267,16 @@ def compile_scc_to_ir(
 
             # Switch to lower abstraction level IR.
             lower_ir(fn, compiler_options)
+            # Perform optimizations that may introduce new primitives with
+            # their own source/header dependencies (e.g. str_extra_ops.h).
+            # Run them before dependency collection so their needs are picked up.
+            do_char_str_index_fold(fn, compiler_options)
+            do_str_buffer_hoist(fn, compiler_options)
             # Calculate implicit module dependencies (needed for librt)
             deps = find_implicit_op_dependencies(fn)
             if deps is not None:
                 module.dependencies.update(deps)
-            # Perform optimizations.
+            # Remaining optimizations.
             do_copy_propagation(fn, compiler_options)
             do_flag_elimination(fn, compiler_options)
 

mypyc/ir/rtypes.py

@@ -394,6 +394,28 @@ def __hash__(self) -> int:
     error_overlap=True,
 )
 
+# char represents a single Unicode codepoint. Stored unboxed as int32 (the
+# full Unicode codepoint range of 0..0x10FFFF fits comfortably). Boxed form
+# is a 1-character str. Distinct from int32_rprimitive so specializers can
+# route char-typed operations through codepoint primitives.
+char_rprimitive: Final = RPrimitive(
+    "char",
+    is_unboxed=True,
+    is_refcounted=False,
+    # Marked native_int so mypyc's fixed-width coercion paths handle
+    # char <-> int32 transparently. Semantically char is a codepoint rather
+    # than a general integer; users shouldn't do arithmetic on it.
+    is_native_int=True,
+    is_signed=True,
+    ctype="int32_t",
+    size=4,
+    # error_overlap=False: the error sentinel -113 is outside the valid Unicode
+    # codepoint range (0..0x10FFFF), so the sentinel compare alone is
+    # authoritative — no PyErr_Occurred confirmation needed. This eliminates
+    # the double error check that ERR_MAGIC_OVERLAPPING requires.
+    error_overlap=False,
+)
+
 # The following unsigned native int types (u16, u32, u64) are not
 # exposed to the user. They are for internal use within mypyc only.
 
@@ -592,13 +614,18 @@ def is_fixed_width_rtype(rtype: RType) -> TypeGuard[RPrimitive]:
         or is_int32_rprimitive(rtype)
         or is_int16_rprimitive(rtype)
         or is_uint8_rprimitive(rtype)
+        or is_char_rprimitive(rtype)
     )
 
 
 def is_uint8_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]:
     return rtype is uint8_rprimitive
 
 
+def is_char_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]:
+    return rtype is char_rprimitive
+
+
 def is_uint32_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]:
     return rtype is uint32_rprimitive
 

mypyc/irbuild/expression.py

@@ -81,8 +81,10 @@
     RTuple,
     RVec,
     bool_rprimitive,
+    char_rprimitive,
     int_rprimitive,
     is_any_int,
+    is_char_rprimitive,
     is_fixed_width_rtype,
     is_int64_rprimitive,
     is_int_rprimitive,
@@ -91,6 +93,7 @@
     is_object_rprimitive,
     object_rprimitive,
     set_rprimitive,
+    short_int_rprimitive,
 )
 from mypyc.irbuild.ast_helpers import is_borrow_friendly_expr, process_conditional
 from mypyc.irbuild.builder import IRBuilder, int_borrow_friendly_op
@@ -115,6 +118,7 @@
     apply_method_specialization,
     translate_object_new,
     translate_object_setattr,
+    try_emit_str_index_as_int,
 )
 from mypyc.irbuild.vec import (
     vec_append,
@@ -838,6 +842,63 @@ def precompute_set_literal(builder: IRBuilder, s: SetExpr) -> Value | None:
     return None
 
 
+def try_specialize_str_char_compare(
+    builder: IRBuilder, op: str, lhs: Expression, rhs: Expression, line: int
+) -> Value | None:
+    """Specialize ``s[i] == 'x'`` / ``s[i] != 'x'`` where ``'x'`` is a 1-char
+    string literal and ``s: str``, ``i`` is int-like.
+
+    Rewrites the comparison to an integer compare of the codepoint, avoiding
+    allocation of a single-char PyObject and the full PyUnicode_Compare path.
+    """
+    for index_side, literal_side in ((lhs, rhs), (rhs, lhs)):
+        if not isinstance(index_side, IndexExpr):
+            continue
+        folded = constant_fold_expr(builder, literal_side)
+        if not isinstance(folded, str) or len(folded) != 1:
+            continue
+        char_int = try_emit_str_index_as_int(builder, index_side)
+        if char_int is None:
+            continue
+        literal_int = Integer(ord(folded), short_int_rprimitive, line)
+        return builder.binary_op(char_int, literal_int, op, line)
+    return None
+
+
+def try_specialize_char_compare(
+    builder: IRBuilder, op: str, lhs: Expression, rhs: Expression, line: int
+) -> Value | None:
+    """Specialize ``char == char`` and ``char == 1-char-str-literal``
+    (and the != variants, and symmetric orderings) to int compare of the
+    underlying codepoint.
+    """
+    if op not in ("==", "!="):
+        return None
+    lhs_type = builder.node_type(lhs)
+    rhs_type = builder.node_type(rhs)
+    # char == char
+    if is_char_rprimitive(lhs_type) and is_char_rprimitive(rhs_type):
+        l_val = builder.accept(lhs)
+        r_val = builder.accept(rhs)
+        return builder.binary_op(l_val, r_val, op, line)
+    # char == <0- or 1-char str literal> (and symmetric). Empty string "" is
+    # encoded as the char empty sentinel -1; any 1-char literal is its ord.
+    for char_side, char_type, lit_side in (
+        (lhs, lhs_type, rhs),
+        (rhs, rhs_type, lhs),
+    ):
+        if not is_char_rprimitive(char_type):
+            continue
+        folded = constant_fold_expr(builder, lit_side)
+        if not isinstance(folded, str) or len(folded) > 1:
+            continue
+        char_val = builder.accept(char_side)
+        codepoint_val = -1 if len(folded) == 0 else ord(folded)
+        codepoint = Integer(codepoint_val, char_rprimitive, line)
+        return builder.binary_op(char_val, codepoint, op, line)
+    return None
+
+
 def transform_comparison_expr(builder: IRBuilder, e: ComparisonExpr) -> Value:
     # x in (...)/[...]
     # x not in (...)/[...]
@@ -848,6 +909,15 @@ def transform_comparison_expr(builder: IRBuilder, e: ComparisonExpr) -> Value:
             return result
 
     if len(e.operators) == 1:
+        # Codepoint fast paths for equality:
+        #   char == char / char == "c"   -> int compare
+        #   s[i] == "c"                  -> int compare against codepoint
+        if first_op in ("==", "!="):
+            for specializer in (try_specialize_char_compare, try_specialize_str_char_compare):
+                result = specializer(builder, first_op, e.operands[0], e.operands[1], e.line)
+                if result is not None:
+                    return result
+
         # Special some common simple cases
         if first_op in ("is", "is not"):
             right_expr = e.operands[1]

mypyc/irbuild/ll_builder.py

@@ -104,6 +104,7 @@
     is_bool_or_bit_rprimitive,
     is_bytes_rprimitive,
     is_c_py_ssize_t_rprimitive,
+    is_char_rprimitive,
     is_dict_rprimitive,
     is_fixed_width_rtype,
     is_float_rprimitive,
@@ -2126,6 +2127,12 @@ def bool_value(self, value: Value) -> Value:
         elif is_runtime_subtype(value.type, int_rprimitive):
             zero = Integer(0, short_int_rprimitive)
             result = self.comparison_op(value, zero, ComparisonOp.NEQ, value.line)
+        elif is_char_rprimitive(value.type):
+            # char is falsy only for the empty sentinel (-1). A codepoint of
+            # 0 (NUL) is a valid non-empty char and must be truthy, matching
+            # str bool semantics where ``"\0"`` is truthy.
+            empty = Integer(-1, value.type)
+            result = self.add(ComparisonOp(value, empty, ComparisonOp.NEQ))
         elif is_fixed_width_rtype(value.type):
             zero = Integer(0, value.type)
             result = self.add(ComparisonOp(value, zero, ComparisonOp.NEQ))