[mypyc] Add native char type + codepoint fast paths for str ops

Adds a first-class `char` native type to mypyc, modeled on i64: stored
unboxed as int32 codepoint, with -1 as the empty-string sentinel, and
bidirectional str<->char promotion. Unblocks codepoint-level fast paths
in per-char loops.

Core type plumbing:
- MYPYC_NATIVE_CHAR_NAMES alongside MYPYC_NATIVE_INT_NAMES
- str <-> char bidirectional _promote in semanal_classprop
- str covers char in subtypes.covers_at_runtime + overlap in meet
- char_rprimitive (int32, is_native_int, error_overlap=False)
- mypy_extensions.char stub

Boxing / unboxing:
- CPyChar_FromObject (accepts 0/1-char str, -113 on type error)
- CPyChar_ToStr (uses interned empty-str singleton for -1)
- bool(char) checks != -1, not != 0, so "\0" stays truthy

Codegen fast paths:
- try_specialize_codepoint_compare in transform_comparison_expr handles
  char/char, char/s[i], char/0-or-1-char-literal, and s[i]/literal
  uniformly, compiling to int compare of the codepoint
- ord(s[i]) refactored to share the codepoint read path
- char.isspace/isdigit/isalnum/isalpha/isidentifier/upper method_ops
  route to codepoint-taking C helpers in str_extra_ops.h
- CPyChar_IsIdentifier delegates to PyUnicode_IsIdentifier for non-ASCII
  (correct XID_Start handling rather than Py_UNICODE_ISALPHA approximation)
- CPyChar_Upper falls back to str.upper() for non-ASCII, returning the
  original codepoint when upper() produces multiple chars (e.g. ß -> SS)
  since char holds one codepoint

New IR transform pass (runs after lower_ir, before dep collection):
- char_str_index_fold: folds Unbox(CPyStr_GetItem(s, i) -> char) to a
  direct CPyStr_GetCharAt int32 read, avoiding the 1-char PyObject alloc

Also adds str.isalpha() method_op via CPyStr_IsAlpha.

Tests:
- run-char.test covers boxing/unboxing, bool semantics (NUL is truthy,
  empty is falsy), equality, classification methods (including non-ASCII
  XID_Start for isidentifier), upper (including ß -> ß pinning for the
  multi-char fallback), str promotion, concatenation, s[i]=="x"
  specialization, ord, and astral-plane codepoints.
- char stub added to test-data/unit/lib-stub/mypy_extensions.pyi so the
  test harness can resolve the type.

Author: VaggelisD

mypy/meet.py

@@ -17,6 +17,7 @@
 )
 from mypy.typeops import is_recursive_pair, make_simplified_union, tuple_fallback
 from mypy.types import (
+    MYPYC_NATIVE_CHAR_NAMES,
     MYPYC_NATIVE_INT_NAMES,
     TUPLE_LIKE_INSTANCE_NAMES,
     AnyType,
@@ -592,6 +593,8 @@ def _type_object_overlap(left: Type, right: Type) -> bool:
 
         if right.type.fullname == "builtins.int" and left.type.fullname in MYPYC_NATIVE_INT_NAMES:
             return True
+        if right.type.fullname == "builtins.str" and left.type.fullname in MYPYC_NATIVE_CHAR_NAMES:
+            return True
 
         # Two unrelated types cannot be partially overlapping: they're disjoint.
         if left.type.has_base(right.type.fullname):

mypy/semanal_classprop.py

@@ -22,7 +22,7 @@
     Var,
 )
 from mypy.options import Options
-from mypy.types import MYPYC_NATIVE_INT_NAMES, Instance, ProperType
+from mypy.types import MYPYC_NATIVE_CHAR_NAMES, MYPYC_NATIVE_INT_NAMES, Instance, ProperType
 
 # Hard coded type promotions (shared between all Python versions).
 # These add extra ad-hoc edges to the subtyping relation. For example,
@@ -184,5 +184,11 @@ def add_type_promotion(
         assert isinstance(int_sym.node, TypeInfo)
         int_sym.node._promote.append(Instance(defn.info, []))
         defn.info.alt_promote = Instance(int_sym.node, [])
+    # Same pattern for str <-> char (mypyc native char type).
+    if defn.fullname in MYPYC_NATIVE_CHAR_NAMES:
+        str_sym = builtin_names["str"]
+        assert isinstance(str_sym.node, TypeInfo)
+        str_sym.node._promote.append(Instance(defn.info, []))
+        defn.info.alt_promote = Instance(str_sym.node, [])
     if promote_targets:
         defn.info._promote.extend(promote_targets)

mypy/subtypes.py

@@ -36,6 +36,7 @@
 from mypy.options import Options
 from mypy.state import state
 from mypy.types import (
+    MYPYC_NATIVE_CHAR_NAMES,
     MYPYC_NATIVE_INT_NAMES,
     TUPLE_LIKE_INSTANCE_NAMES,
     TYPED_NAMEDTUPLE_NAMES,
@@ -2166,6 +2167,10 @@ def covers_at_runtime(item: Type, supertype: Type) -> bool:
             # "int" covers all native int types
             if item.type.fullname in MYPYC_NATIVE_INT_NAMES:
                 return True
+        elif isinstance(item, Instance) and supertype.type.fullname == "builtins.str":
+            # "str" covers the native char type
+            if item.type.fullname in MYPYC_NATIVE_CHAR_NAMES:
+                return True
     # TODO: Add more special cases.
     return False
 

mypy/types.py

@@ -190,6 +190,10 @@
     "mypy_extensions.u8",
 )
 
+# Mypyc native char type (compatible with builtins.str, bidirectionally;
+# stored unboxed as an int32 codepoint with -1 as the empty sentinel).
+MYPYC_NATIVE_CHAR_NAMES: Final = ("mypy_extensions.char",)
+
 DATACLASS_TRANSFORM_NAMES: Final = (
     "typing.dataclass_transform",
     "typing_extensions.dataclass_transform",

mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi

@@ -220,3 +220,27 @@ class u8:
     def __gt__(self, x: u8) -> bool: ...
     def __index__(self) -> int: ...
     def __eq__(self, x: object) -> bool: ...
+
+# char represents at most one Unicode codepoint. At the type-check level it's
+# bidirectionally compatible with str via the mypyc native-string promotion
+# mechanism. Under mypyc a char is stored unboxed as an int32 codepoint
+# (with -1 for the empty sentinel).
+class char:
+    def __new__(cls, __x: str = ...) -> char: ...
+    def __eq__(self, x: object) -> bool: ...
+    def __ne__(self, x: object) -> bool: ...
+    def __hash__(self) -> int: ...
+    # Mixed char/str concat. Result is str.
+    def __add__(self, x: str) -> str: ...
+    def __radd__(self, x: str) -> str: ...
+    def isspace(self) -> bool: ...
+    def isdigit(self) -> bool: ...
+    def isalnum(self) -> bool: ...
+    def isalpha(self) -> bool: ...
+    def isidentifier(self) -> bool: ...
+    # Case conversion. Returns the original codepoint if the Unicode
+    # result is multi-char (e.g. ß.upper() = "SS") since char holds one
+    # codepoint.
+    def upper(self) -> char: ...
+    def lower(self) -> char: ...
+    def strip(self, __chars: str | None = ...) -> str: ...

mypyc/codegen/emit.py

@@ -44,6 +44,7 @@
     is_bool_or_bit_rprimitive,
     is_bytearray_rprimitive,
     is_bytes_rprimitive,
+    is_char_rprimitive,
     is_dict_rprimitive,
     is_fixed_width_rtype,
     is_float_rprimitive,
@@ -1045,6 +1046,13 @@ def emit_unbox(
             self.emit_line(f"{dest} = CPyLong_AsInt16({src});")
             if not isinstance(error, AssignHandler):
                 self.emit_unbox_failure_with_overlapping_error_value(dest, typ, failure)
+        elif is_char_rprimitive(typ):
+            assert not optional
+            if declare_dest:
+                self.emit_line(f"int32_t {dest};")
+            self.emit_line(f"{dest} = CPyChar_FromObject({src});")
+            if not isinstance(error, AssignHandler):
+                self.emit_unbox_failure_with_overlapping_error_value(dest, typ, failure)
         elif is_uint8_rprimitive(typ):
             # Whether we are borrowing or not makes no difference.
             assert not optional  # Not supported for overlapping error values
@@ -1192,6 +1200,8 @@ def emit_box(
                 self.emit_inc_ref(dest, object_rprimitive)
         elif is_int32_rprimitive(typ) or is_int16_rprimitive(typ) or is_uint8_rprimitive(typ):
             self.emit_line(f"{declaration}{dest} = PyLong_FromLong({src});")
+        elif is_char_rprimitive(typ):
+            self.emit_line(f"{declaration}{dest} = CPyChar_ToStr({src});")
         elif is_int64_rprimitive(typ):
             self.emit_line(f"{declaration}{dest} = PyLong_FromLongLong({src});")
         elif is_float_rprimitive(typ):

mypyc/codegen/emitmodule.py

@@ -66,6 +66,7 @@
 from mypyc.irbuild.prepare import load_type_map
 from mypyc.namegen import NameGenerator, exported_name
 from mypyc.options import CompilerOptions
+from mypyc.transform.char_str_index_fold import do_char_str_index_fold
 from mypyc.transform.copy_propagation import do_copy_propagation
 from mypyc.transform.exceptions import insert_exception_handling
 from mypyc.transform.flag_elimination import do_flag_elimination
@@ -265,11 +266,14 @@ def compile_scc_to_ir(
 
             # Switch to lower abstraction level IR.
             lower_ir(fn, compiler_options)
+            # Run char_str_index_fold before dependency collection so the new
+            # str_extra_ops.h primitives it introduces are picked up.
+            do_char_str_index_fold(fn, compiler_options)
             # Calculate implicit module dependencies (needed for librt)
             deps = find_implicit_op_dependencies(fn)
             if deps is not None:
                 module.dependencies.update(deps)
-            # Perform optimizations.
+            # Remaining optimizations.
             do_copy_propagation(fn, compiler_options)
             do_flag_elimination(fn, compiler_options)
 

mypyc/ir/rtypes.py

@@ -394,6 +394,20 @@ def __hash__(self) -> int:
     error_overlap=True,
 )
 
+# char: single Unicode codepoint stored as int32; -1 empty sentinel, -113
+# error sentinel. Distinct from int32_rprimitive so specializers can route
+# char-typed operations through codepoint primitives.
+char_rprimitive: Final = RPrimitive(
+    "char",
+    is_unboxed=True,
+    is_refcounted=False,
+    is_native_int=True,
+    is_signed=True,
+    ctype="int32_t",
+    size=4,
+    error_overlap=False,
+)
+
 # The following unsigned native int types (u16, u32, u64) are not
 # exposed to the user. They are for internal use within mypyc only.
 
@@ -592,13 +606,18 @@ def is_fixed_width_rtype(rtype: RType) -> TypeGuard[RPrimitive]:
         or is_int32_rprimitive(rtype)
         or is_int16_rprimitive(rtype)
         or is_uint8_rprimitive(rtype)
+        or is_char_rprimitive(rtype)
     )
 
 
 def is_uint8_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]:
     return rtype is uint8_rprimitive
 
 
+def is_char_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]:
+    return rtype is char_rprimitive
+
+
 def is_uint32_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]:
     return rtype is uint32_rprimitive
 

mypyc/irbuild/expression.py

@@ -79,18 +79,24 @@
 from mypyc.ir.rtypes import (
     RInstance,
     RTuple,
+    RType,
     RVec,
     bool_rprimitive,
+    char_rprimitive,
     int_rprimitive,
     is_any_int,
+    is_char_rprimitive,
     is_fixed_width_rtype,
     is_int64_rprimitive,
     is_int_rprimitive,
     is_list_rprimitive,
     is_none_rprimitive,
     is_object_rprimitive,
+    is_str_rprimitive,
+    is_tagged,
     object_rprimitive,
     set_rprimitive,
+    short_int_rprimitive,
 )
 from mypyc.irbuild.ast_helpers import is_borrow_friendly_expr, process_conditional
 from mypyc.irbuild.builder import IRBuilder, int_borrow_friendly_op
@@ -115,6 +121,7 @@
     apply_method_specialization,
     translate_object_new,
     translate_object_setattr,
+    try_emit_str_index_as_int,
 )
 from mypyc.irbuild.vec import (
     vec_append,
@@ -843,6 +850,66 @@ def precompute_set_literal(builder: IRBuilder, s: SetExpr) -> Value | None:
     return None
 
 
+def _codepoint_kind(builder: IRBuilder, expr: Expression, expr_type: RType) -> str | None:
+    """Classify expr as a codepoint candidate without emitting IR.
+
+    Returns "char", "index" (for ``s[i]`` with int-like index), or None.
+    """
+    if is_char_rprimitive(expr_type):
+        return "char"
+    if isinstance(expr, IndexExpr) and is_str_rprimitive(builder.node_type(expr.base)):
+        idx_type = builder.node_type(expr.index)
+        if is_tagged(idx_type) or is_fixed_width_rtype(idx_type):
+            return "index"
+    return None
+
+
+def _emit_codepoint_value(
+    builder: IRBuilder, expr: Expression, kind: str
+) -> tuple[Value, RType]:
+    """Emit the codepoint read. ``kind`` must come from _codepoint_kind."""
+    if kind == "char":
+        return builder.accept(expr), char_rprimitive
+    assert isinstance(expr, IndexExpr)
+    val = try_emit_str_index_as_int(builder, expr)
+    assert val is not None  # _codepoint_kind guarantees this
+    return val, short_int_rprimitive
+
+
+def try_specialize_codepoint_compare(
+    builder: IRBuilder, op: str, lhs: Expression, rhs: Expression, line: int
+) -> Value | None:
+    """Rewrite ``x == y`` / ``x != y`` to an int compare of codepoints when at
+    least one side is a codepoint (``char`` value or ``s[i]`` on a str) and the
+    other is either another codepoint or a 0/1-char str literal. Avoids the
+    1-char PyObject alloc + PyUnicode_Compare.
+    """
+    if op not in ("==", "!="):
+        return None
+    lhs_kind = _codepoint_kind(builder, lhs, builder.node_type(lhs))
+    rhs_kind = _codepoint_kind(builder, rhs, builder.node_type(rhs))
+    if lhs_kind is None and rhs_kind is None:
+        return None
+    # Codepoint on both sides: direct int compare.
+    if lhs_kind is not None and rhs_kind is not None:
+        l_val, _ = _emit_codepoint_value(builder, lhs, lhs_kind)
+        r_val, _ = _emit_codepoint_value(builder, rhs, rhs_kind)
+        return builder.binary_op(l_val, r_val, op, line)
+    # One side codepoint, other side must fold to a 0/1-char str literal.
+    if lhs_kind is not None:
+        cp_expr, cp_kind, lit_expr = lhs, lhs_kind, rhs
+    else:
+        assert rhs_kind is not None
+        cp_expr, cp_kind, lit_expr = rhs, rhs_kind, lhs
+    folded = constant_fold_expr(builder, lit_expr)
+    if not isinstance(folded, str) or len(folded) > 1:
+        return None  # No IR emitted yet — safe to bail.
+    val, rtype = _emit_codepoint_value(builder, cp_expr, cp_kind)
+    # Empty string encodes as -1 (char empty sentinel).
+    codepoint = -1 if len(folded) == 0 else ord(folded)
+    return builder.binary_op(val, Integer(codepoint, rtype, line), op, line)
+
+
 def transform_comparison_expr(builder: IRBuilder, e: ComparisonExpr) -> Value:
     # x in (...)/[...]
     # x not in (...)/[...]
@@ -853,6 +920,15 @@ def transform_comparison_expr(builder: IRBuilder, e: ComparisonExpr) -> Value:
             return result
 
     if len(e.operators) == 1:
+        # Codepoint fast path: char/char, char/s[i], or codepoint/1-char-literal
+        # -> int compare instead of PyUnicode_Compare.
+        if first_op in ("==", "!="):
+            result = try_specialize_codepoint_compare(
+                builder, first_op, e.operands[0], e.operands[1], e.line
+            )
+            if result is not None:
+                return result
+
         # Special some common simple cases
         if first_op in ("is", "is not"):
             right_expr = e.operands[1]

mypyc/irbuild/ll_builder.py

@@ -104,6 +104,7 @@
     is_bool_or_bit_rprimitive,
     is_bytes_rprimitive,
     is_c_py_ssize_t_rprimitive,
+    is_char_rprimitive,
     is_dict_rprimitive,
     is_fixed_width_rtype,
     is_float_rprimitive,
@@ -685,6 +686,7 @@ def coerce_short_int_to_fixed_width(self, src: Value, target_type: RType, line:
     def coerce_fixed_width_to_int(self, src: Value, line: int) -> Value:
         if (
             (is_int32_rprimitive(src.type) and PLATFORM_SIZE == 8)
+            or (is_char_rprimitive(src.type) and PLATFORM_SIZE == 8)
             or is_int16_rprimitive(src.type)
             or is_uint8_rprimitive(src.type)
         ):
@@ -717,7 +719,7 @@ def coerce_fixed_width_to_int(self, src: Value, line: int) -> Value:
         self.activate_block(slow)
         if is_int64_rprimitive(src_type):
             conv_op = int64_to_int_op
-        elif is_int32_rprimitive(src_type):
+        elif is_int32_rprimitive(src_type) or is_char_rprimitive(src_type):
             assert PLATFORM_SIZE == 4
             conv_op = ssize_t_to_int_op
         else:
@@ -2126,6 +2128,12 @@ def bool_value(self, value: Value) -> Value:
         elif is_runtime_subtype(value.type, int_rprimitive):
             zero = Integer(0, short_int_rprimitive)
             result = self.comparison_op(value, zero, ComparisonOp.NEQ, value.line)
+        elif is_char_rprimitive(value.type):
+            # char is falsy only for the empty sentinel (-1). A codepoint of
+            # 0 (NUL) is a valid non-empty char and must be truthy, matching
+            # str bool semantics where ``"\0"`` is truthy.
+            empty = Integer(-1, value.type)
+            result = self.add(ComparisonOp(value, empty, ComparisonOp.NEQ))
         elif is_fixed_width_rtype(value.type):
             zero = Integer(0, value.type)
             result = self.add(ComparisonOp(value, zero, ComparisonOp.NEQ))