Update Rust detection and binding generation

This commit updates the build system to automatically detect cargo and
enable/disable _base64 without needing to pass a flag.

It also updates cpython-sys to use a hand written header (which is what
Linux seems to do) and splits off the parser bindings to be handled in
the future (since the files are included differently).

Author: emmatyping

Makefile.pre.in

@@ -1652,10 +1652,6 @@ Makefile Modules/config.c: Makefile.pre \
 	@mv config.c Modules
 	@echo "The Makefile was updated, you may need to re-run make."
 
-.PHONY: regen-rust-wrapper-h
-regen-rust-wrapper-h: $(PYTHON_HEADERS)
-	PYTHON_HEADERS="$(PYTHON_HEADERS)" $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/regen-rust-wrapper-h.py
-
 .PHONY: regen-test-frozenmain
 regen-test-frozenmain: $(BUILDPYTHON)
 	# Regenerate Programs/test_frozenmain.h
@@ -3376,6 +3372,9 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
 ##########################################################################
 # Module dependencies and platform-specific files
 
+cpython-sys: Modules/cpython-sys/Cargo.toml Modules/cpython-sys/build.rs Modules/cpython-sys/wrapper.h Modules/cpython-sys/parser.h
+	cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE)
+
 # force rebuild when header file or module build flavor (static/shared) is changed
 MODULE_DEPS_STATIC=Modules/config.c
 MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@

Modules/cpython-sys/build.rs

@@ -4,25 +4,29 @@ use std::path::{Path, PathBuf};
 fn main() {
     let curdir = std::env::current_dir().unwrap();
     let srcdir = curdir.parent().and_then(Path::parent).unwrap();
+    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    generate_c_api_bindings(srcdir, &out_path.as_path());
+    // TODO(emmatyping): generate bindings to the internal parser API
+    // The parser includes things slightly differently, so we should generate
+    // it's bindings independently
+    //generate_parser_bindings(srcdir, &out_path.as_path());
+}
+
+fn generate_c_api_bindings(srcdir: &Path, out_path: &Path) {
     let bindings = bindgen::Builder::default()
         .header("wrapper.h")
         .clang_arg(format!("-I{}", srcdir.as_os_str().to_str().unwrap()))
         .clang_arg(format!("-I{}/Include", srcdir.as_os_str().to_str().unwrap()))
-        .clang_arg(format!("-I{}/Include/internal", srcdir.as_os_str().to_str().unwrap()))
-        .allowlist_function("Py.*")
-        .allowlist_function("_Py.*")
-        .allowlist_type("Py.*")
-        .allowlist_type("_Py.*")
-        .allowlist_var("Py.*")
-        .allowlist_var("_Py.*")
+        .allowlist_function("_?Py.*")
+        .allowlist_type("_?Py.*")
+        .allowlist_var("_?Py.*")
         .blocklist_type("^PyMethodDef$")
         .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
         .generate()
         .expect("Unable to generate bindings");
 
-    // Write the bindings to the $OUT_DIR/bindings.rs file.
-    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    // Write the bindings to the $OUT_DIR/c_api.rs file.
     bindings
-        .write_to_file(out_path.join("bindings.rs"))
+        .write_to_file(out_path.join("c_api.rs"))
         .expect("Couldn't write bindings!");
-}
+}

Modules/cpython-sys/parser.h

@@ -0,0 +1,11 @@
+/* Private APIs */
+#define Py_BUILD_CORE
+
+// Parser
+#include "Parser/pegen.h"
+#include "Parser/string_parser.h"
+#include "Parser/lexer/buffer.h"
+#include "Parser/lexer/lexer.h"
+#include "Parser/lexer/state.h"
+#include "Parser/tokenizer/tokenizer.h"
+#include "Parser/tokenizer/helpers.h"

Modules/cpython-sys/src/lib.rs

@@ -6,7 +6,10 @@
 
 use std::ffi::{c_char, c_int, c_void};
 
-include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+include!(concat!(env!("OUT_DIR"), "/c_api.rs"));
+
+// TODO(emmatyping): include parser bindings (see build.rs)
+//include!(concat!(env!("OUT_DIR"), "/parser.rs"));
 /* Flag passed to newmethodobject */
 /* #define METH_OLDARGS  0x0000   -- unsupported now */
 pub const METH_VARARGS: c_int = 0x0001;

Modules/makesetup

@@ -286,7 +286,7 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 			done
 			libs=
 			# depends on the headers through cpython-sys
-			rule="$objs: \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest Modules/cpython-sys/wrapper.h $prefixed_srcs \$(PYTHON_HEADERS)"
+			rule="$objs: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS)"
 			rule="$rule; cargo build --lib --locked --package ${mods} --profile \$(CARGO_PROFILE)"
 			echo "$rule" >>$rulesf
 			for mod in $mods

Python/remote_debug.h

@@ -29,7 +29,6 @@ extern "C" {
 
 #include "pyconfig.h"
 #include "internal/pycore_ceval.h"
-#include "internal/pycore_debug_offsets.h"
 
 #ifdef __linux__
 #    include <elf.h>

Tools/build/regen-rust-wrapper-h.py

@@ -4309,40 +4309,29 @@ AC_SUBST([LIBMPDEC_INTERNAL])
 
 dnl Try to detect cargo in the environment. Cargo and rustup
 dnl install into CARGO_HOME and RUSTUP_HOME, so check for those initially
-AC_MSG_CHECKING([for --with-rust-base64])
-AC_ARG_WITH(
-  [rust_base64],
-  [AS_HELP_STRING(
-    [--with-rust-base64],
-    [build _base64 module using the SIMD accelerated Rust implementation]
-  )],
-  [rust_base64="yes"],
-  [rust_base64="no"])
-AC_MSG_RESULT([$rust_base64])
-
-AS_VAR_IF(
-  [rust_base64], [yes],
-  [
-    if test "$CARGO_HOME+set" != "set"; then
-      dnl try to guess the default UNIX value of ~/.cargo
-      CARGO_HOME="$HOME/.cargo"
-    fi
-    AC_CHECK_PROG(HAS_CARGO, [cargo], ["$CARGO_HOME"], [found], [not-found])
-    if test $HAS_CARGO = "not-found"; then
-      AC_MSG_ERROR([Could not find cargo. Please re-run configure with \$CARGO_HOME set])
-    fi
-    if test "$Py_OPT" = 'true'; then
-      CARGO_TARGET_DIR='release'
-      CARGO_PROFILE='release'
-    else
-      CARGO_TARGET_DIR='debug'
-      CARGO_PROFILE='dev'
-    fi
-    AC_SUBST([CARGO_HOME])
-    AC_SUBST([CARGO_TARGET_DIR])
-    AC_SUBST([CARGO_PROFILE])
-  ]
-)
+AC_MSG_CHECKING([for Rust])
+CARGO_TARGET_DIR=
+CARGO_PROFILE=
+if test "x$CARGO_HOME" = "x"; then
+  dnl try to guess the default UNIX value of ~/.cargo
+  CARGO_HOME="$HOME/.cargo"
+fi
+AC_CHECK_PROG(HAS_CARGO, [cargo], ["$CARGO_HOME"], [found], [not-found])
+if test $HAS_CARGO = "not-found"; then
+  AC_MSG_RESULT([n/a])
+  AC_MSG_WARN([Could not find the cargo executable. It can be installed via rustup])
+else
+  if test "$Py_OPT" = 'true'; then
+    CARGO_TARGET_DIR='release'
+    CARGO_PROFILE='release'
+  else
+    CARGO_TARGET_DIR='debug'
+    CARGO_PROFILE='dev'
+  fi
+fi
+AC_SUBST([CARGO_HOME])
+AC_SUBST([CARGO_TARGET_DIR])
+AC_SUBST([CARGO_PROFILE])
 
 
 dnl detect sqlite3 from Emscripten emport
@@ -8194,7 +8183,7 @@ PY_STDLIB_MOD([_uuid],
   [$LIBUUID_CFLAGS], [$LIBUUID_LIBS])
 
 PY_STDLIB_MOD([_base64],
-  [], [test "$rust_base64" = "yes"],
+  [], [test "$HAS_CARGO" = "found"],
   [], [])
 
 dnl compression libs