From a0d859e93662b4e46fff5471212d001f3efc2d7a Mon Sep 17 00:00:00 2001 From: Charlie <5764343+charlielye@users.noreply.github.com> Date: Mon, 15 Jun 2026 10:11:21 +0000 Subject: [PATCH] feat(ipc-codegen): human-authored JSONC schema format Add a friendly JSONC schema front-end to schema_visitor.ts that lowers a single per-service object (service/aliases/types/error/commands with shorthand string type refs) to the positional named_union form the generators already consume, so the four generators are untouched and the produced CompiledSchema is identical. The type prefix and method-prefix stripping fold into the 'service' field, retiring --prefix/--strip-method-prefix and detectPrefix for friendly schemas. aliases support bin32 (nominal) and scalar synonyms (e.g. MerkleTreeId = u32). Convert the echo example schema to schema.jsonc and update its bootstraps; remove the schema_reflection_test (it asserted the old positional shape). Add scripts/convert_schema.ts, a one-shot positional->friendly converter used to migrate the remaining service schemas. Goldens unchanged; full echo test matrix (golden x4, UDS 4x4, SHM, ts_package) passes. --- ipc-codegen/SCHEMA_SPEC.md | 350 ++++++++---------- ipc-codegen/bootstrap.sh | 1 - ipc-codegen/echo_example/cpp/CMakeLists.txt | 3 - ipc-codegen/echo_example/cpp/bootstrap.sh | 6 +- .../cpp/src/schema_reflection_test.cpp | 79 ---- ipc-codegen/echo_example/rust/bootstrap.sh | 6 +- ipc-codegen/echo_example/schema/schema.json | 118 ------ ipc-codegen/echo_example/schema/schema.jsonc | 44 +++ ipc-codegen/echo_example/ts/bootstrap.sh | 5 +- .../echo_example/ts/src/echo_client.ts | 20 +- .../echo_example/ts/src/echo_server.ts | 12 +- .../echo_example/ts_package/bootstrap.sh | 4 +- ipc-codegen/echo_example/zig/bootstrap.sh | 6 +- ipc-codegen/scripts/convert_schema.ts | 173 +++++++++ ipc-codegen/src/cpp_codegen.ts | 55 +-- ipc-codegen/src/generate.ts | 52 ++- ipc-codegen/src/rust_codegen.ts | 18 +- ipc-codegen/src/schema_visitor.ts | 170 +++++++++ .../templates/cpp/ipc_codegen/named_union.hpp | 21 +- .../templates/cpp/ipc_codegen/schema.hpp | 214 ----------- ipc-codegen/test/schema_visitor.test.ts | 13 +- 21 files changed, 634 insertions(+), 736 deletions(-) delete mode 100644 ipc-codegen/echo_example/cpp/src/schema_reflection_test.cpp delete mode 100644 ipc-codegen/echo_example/schema/schema.json create mode 100644 ipc-codegen/echo_example/schema/schema.jsonc create mode 100644 ipc-codegen/scripts/convert_schema.ts delete mode 100644 ipc-codegen/templates/cpp/ipc_codegen/schema.hpp diff --git a/ipc-codegen/SCHEMA_SPEC.md b/ipc-codegen/SCHEMA_SPEC.md index 3107bf44e490..79f75956e935 100644 --- a/ipc-codegen/SCHEMA_SPEC.md +++ b/ipc-codegen/SCHEMA_SPEC.md @@ -1,159 +1,158 @@ # IPC Schema Format Specification -This document specifies the JSON schema format used for cross-language code generation -in the IPC codegen system. The schema is the contract between a producer's -schema export command and all language code generators. +This document specifies the schema format used for cross-language code generation +in the IPC codegen system. A schema is a single hand-authored JSONC file per +service and is the source of truth: ipc-codegen reads it to generate the wire +types, client, and server dispatch for every target language (TypeScript, C++, +Rust, Zig). The committed golden msgpack corpus is the cross-language wire-format +contract; the schema is a normal reviewed source file. -## Overview +JSONC is plain JSON with `//` and `/* */` comments stripped before parsing — no +extra dependencies. -Each IPC service exports its schema as JSON, typically via a subcommand: +## Top-level structure -```bash -./my-service msgpack schema # Outputs JSON to stdout -``` +A schema is a single object describing one service: -The output is a JSON object representing the service's API, derived at compile time -from C++ type metadata via the `MsgpackSchemaPacker` infrastructure. +```jsonc +{ + "service": "Echo", -## Top-Level Structure + // Named byte aliases — nominal 32-byte types. Only bin32 today. + "aliases": { + "Fr": "bin32" + }, -```json -{ - "commands": ["named_union", [ - ["CommandNameA", { "__typename": "CommandNameA", "field1": , ... }], - ["CommandNameB", { "__typename": "CommandNameB", "field1": , ... }] - ]], - "responses": ["named_union", [ - ["ResponseNameA", { "__typename": "ResponseNameA", "field1": , ... }], - ["ErrorResponse", { "__typename": "ErrorResponse", "message": "string" }] - ]] -} -``` + // Shared struct types, referenced by name from commands or other types. + "types": { + "EchoInner": { + "values": "bytes[]", + "flag": "bool?" + } + }, -- `commands` and `responses` are both **NamedUnion** types (see below). -- Commands and responses are paired by name: command `Foo` corresponds to the - response named `FooResponse`. The error response (ending in `ErrorResponse`) - is shared across all commands. + // The error variant, declared once and shared by every command. + "error": { "message": "string" }, -### Validation rules + // command -> { request, response }. + "commands": { + "Bytes": { "request": { "data": "bytes" }, + "response": { "data": "bytes" } }, -Schemas are validated at generation time; violations are hard errors: + "Fields": { "request": { "a": "u32", "b": "u64", "name": "string" }, + "response": { "a": "u32", "b": "u64", "name": "string" } }, -- Exactly one response variant named `*ErrorResponse` must exist, with - exactly one field `message: string`. Generated servers wrap handler - failures into this variant; generated clients surface its message. -- Every command `Foo` must have a matching response `FooResponse`, and the - number of commands must equal the number of non-error responses. -- Command names must be unique. -- Response schemas must be struct definitions, not type-name strings. -- Field names must not map (via the snake_case or camelCase projection) to a - reserved word in any target language, and two field names in one struct - must not collapse to the same projected identifier. -- C++ `SERIALIZATION_FIELDS` supports at most 20 fields per struct. + "Nested": { "request": { "inner": "EchoInner" }, + "response": { "inner": "EchoInner" } }, -## Type Encodings + "Aliases": { "request": { "treeId": "u32", "hash": "Fr", + "maybeHash": "Fr?", "hashes": "Fr[]" }, + "response": { "treeId": "u32", "hash": "Fr", + "maybeHash": "Fr?", "hashes": "Fr[]" } }, -Types in the schema are represented as one of: + "Blobs": { "request": { "maybeData": "bytes?", "parts": "bytes[2]" }, + "response": { "maybeData": "bytes?", "parts": "bytes[2]" } }, -### Primitive Types (JSON strings) + "Fail": { "request": { "message": "string" }, + "response": {} } + } +} +``` -| Schema String | C++ Type | Description | -|---------------|----------|-------------| -| `"bool"` | `bool` | Boolean | -| `"int"` | `int` | Signed 32-bit integer | -| `"unsigned int"` | `unsigned int` / `uint32_t` | Unsigned 32-bit integer | -| `"unsigned short"` | `unsigned short` / `uint16_t` | Unsigned 16-bit integer | -| `"unsigned long"` | `unsigned long` / `uint64_t` | Unsigned 64-bit integer | -| `"unsigned char"` | `unsigned char` / `uint8_t` | Unsigned 8-bit integer | -| `"double"` | `double` | 64-bit floating point | -| `"string"` | `std::string` | UTF-8 string | -| `"bin32"` | `std::array` | Fixed 32-byte binary value | +### `service` -Domain names such as `Fr`, `MerkleTreeId`, `ForkId`, `LeafIndex`, or service-specific IDs are not primitives. Express them as aliases over the primitive wire type. +The service name. It is the prefix for generated **type** names and is *not* +included in **method** names: -### Container Types (JSON arrays) +- Command `Bytes` under `"service": "Echo"` generates the wire type `EchoBytes` + and the response type `EchoBytesResponse`. +- The corresponding client method / server handler is the bare command name + (`bytes` / `handle_bytes`), projected to each language's casing convention. -Container types are encoded as 2-element arrays: `[kind, [args...]]` +The error type is named `ErrorResponse` (e.g. `EchoErrorResponse`). -#### `vector` -```json -["vector", []] -``` -Example: `["vector", ["unsigned char"]]` = `std::vector` = byte array +### `aliases` -**Special case**: `["vector", ["unsigned char"]]` is treated as raw bytes, not an array of integers. +A map of alias name to underlying type. Two kinds: -#### `array` -```json -["array", [, ]] -``` -Example: `["array", ["unsigned char", 32]]` = `std::array` = 32-byte fixed buffer +- **Nominal byte alias** (`bin32`): a distinct named 32-byte value (e.g. `Fr` is + a field element, not raw bytes). It carries its name as a dispatch tag and is + generated as a distinct wrapper type per language. `bin32` is the only nominal + byte width supported today. +- **Scalar synonym**: an alias whose underlying is a primitive (e.g. + `MerkleTreeId: u32`). These are transparent — generated as plain type + aliases — so consumers can `static_cast`/coerce them to and from the + underlying integer or enum. Because they are transparent, declaring them is + optional: a field may simply use the primitive (`u32`) directly. -`["array", ["unsigned char", N]]` is a fixed-length array of integer bytes. Use `"bin32"` or `["alias", ["Name", "bin32"]]` for fixed 32-byte binary values that must encode as msgpack `bin`. +### `types` -#### `optional` -```json -["optional", []] -``` -Example: `["optional", ["string"]]` = `std::optional` +Named shared struct types, each a field-name → type-reference map. A type is +inlined at every reference and deduplicated by name, so it may be referenced +from multiple commands or from other `types`. -#### `shared_ptr` -```json -["shared_ptr", []] -``` -Treated as a transparent wrapper; the inner type is used directly. +### `error` -#### `alias` -```json -["alias", [, ]] -``` -Alias for a named schema type that serializes as a primitive wire type. -The second element must be a primitive schema string. Code generators emit a named type alias over the primitive wire shape. +The error struct, declared once. It must have exactly one field `message` +of type `string`. Generated servers wrap handler failures into this variant; +generated clients surface its `message`. -Examples: +### `commands` -```json -["alias", ["Fr", "bin32"]] -["alias", ["MerkleTreeId", "unsigned int"]] -["alias", ["ForkId", "unsigned long"]] -``` +A map of command name to `{ request, response }`, where each of `request` and +`response` is a field-name → type-reference map. An empty object `{}` denotes a +command with no fields (e.g. a `Fail` command whose response carries nothing). -### Struct Types (JSON objects) +A `response` may instead be a **string** naming another command's response type +to reuse its shape — e.g. `"response": "AliasesResponse"` reuses the +`EchoAliases` response. Use the generated response type name (`Response`). -Structs are JSON objects with a `__typename` field and named fields: +## Type-reference shorthand grammar -```json -{ - "__typename": "SomeStruct", - "field_a": "unsigned int", - "field_b": ["vector", ["unsigned char"]], - "field_c": { - "__typename": "NestedStruct", - "x": "unsigned long" - } -} -``` +Every field type is a shorthand string. The grammar is a leaf type optionally +followed by suffixes, applied right to left: -- `__typename` identifies the struct for deduplication and named reference. -- Field names are the original C++ field names (snake_case by convention). -- Field values are type encodings (primitives, containers, or nested structs). -- Nested structs are inlined on first occurrence and referenced by `__typename` string thereafter. +| Suffix | Meaning | +|---------|-------------------| +| `T?` | optional | +| `T[]` | vector of `T` | +| `T[N]` | fixed array of N | -### NamedUnion Type +Suffixes compose, e.g. `Fr[]`, `bytes?`, `Fr[2]`, `EchoInner[]`. -```json -["named_union", [ - ["VariantName1", ], - ["VariantName2", ] -]] -``` +Leaf types: + +| Leaf | Meaning | +|-----------------|------------------------------------------| +| `bool` | boolean | +| `u8 u16 u32 u64`| unsigned 8/16/32/64-bit integers | +| `f64` | 64-bit float | +| `string` | UTF-8 string | +| `bytes` | variable-length byte string (msgpack bin)| +| `bin32` | fixed 32-byte value (msgpack bin) | +| alias name | a declared `aliases` entry (e.g. `Fr`) | +| type name | a declared `types` entry (e.g. `EchoInner`) | + +## Validation rules + +Schemas are validated at generation time; violations are hard errors: -A tagged union where each variant has a string name and a type schema. -This is the top-level type for both `commands` and `responses`. +- `service` must be a non-empty string. +- The `error` struct must have exactly one field, `message: string`. +- Each command produces a matching `Response`; the command and + non-error response counts must agree. +- Command names must be unique. +- A type reference must resolve to a primitive, a declared alias, or a declared + type. +- Field names must not project (via the snake_case or camelCase mapping) to a + reserved word in any target language, and two fields in one struct must not + collapse to the same projected identifier. +- A struct supports at most 20 fields (the C++ serialization macro limit). -## Wire Protocol +## Wire protocol -The schema defines the types; this section specifies how they are serialized on the wire. +The schema defines the types; this section specifies how a value of each type +is serialized. The golden corpus pins these encodings across all languages. ### Framing @@ -163,96 +162,67 @@ All messages use length-prefix framing: [4 bytes: payload length, little-endian uint32][payload: msgpack bytes] ``` -### Request Wire Format +### Request wire format -A request is a 1-element msgpack **array** containing a NamedUnion: +A request is a 1-element msgpack array wrapping a `[name, payload]` pair: ``` -msgpack array(1) [ - msgpack array(2) [ - msgpack string: "CommandName", - msgpack map: { field1: value1, field2: value2, ... } - ] -] +array(1) [ array(2) [ str: "", map: { field: value, ... } ] ] ``` -In msgpack terms: `[[command_name, {fields...}]]` - -The outer array (tuple wrapper) exists for extensibility. The inner 2-element array -is the NamedUnion encoding. +The dispatch tag is the generated command type name (e.g. `EchoBytes`). The +outer array exists for extensibility. -### Response Wire Format +### Response wire format -A response is a NamedUnion (no tuple wrapper): +A response is a `[name, payload]` pair (no outer wrapper): ``` -msgpack array(2) [ - msgpack string: "ResponseName" | "ErrorResponse", - msgpack map: { field1: value1, field2: value2, ... } -] +array(2) [ str: "Response" | "ErrorResponse", map: { ... } ] ``` -If the response variant name ends with `ErrorResponse`, the response indicates an error. -The error struct always has a `message` field (string). - -### NamedUnion Wire Encoding - -A NamedUnion value is always encoded as a **2-element msgpack array**: -- Element 0: `string` — the variant name (matches `MSGPACK_SCHEMA_NAME` in C++) -- Element 1: `map` — the variant's fields, encoded as a msgpack map with string keys +A response whose name is `ErrorResponse` indicates an error; its +`message` field carries the text. -### Struct Wire Encoding +### Type wire encoding -Structs are encoded as msgpack **maps** with string keys matching the original C++ field names. -The `__typename` field from the schema is NOT included in the wire encoding — it is only -used for schema identification. +| Schema type | msgpack encoding | +|--------------------|-----------------------------------------------| +| `bool` | bool | +| `u8 u16 u32 u64` | integer (smallest encoding that fits) | +| `f64` | float64 | +| `string` | str | +| `bytes` | bin | +| `bin32` | bin (32 bytes) | +| `T?` (optional) | nil if absent, else the encoding of `T` | +| `T[]` (vector) | array | +| `T[N]` (array) | array (fixed length) | +| alias | same encoding as the alias's underlying type | +| struct | map with string keys (field names) | -### Type Wire Encoding Summary +### Integer encoding note -| Schema Type | msgpack Encoding | -|-------------|------------------| -| `bool` | msgpack bool | -| `unsigned int`, `int` | msgpack integer (smallest encoding that fits) | -| `unsigned short` | msgpack integer | -| `unsigned long` | msgpack integer | -| `unsigned char` | msgpack integer | -| `double` | msgpack float64 | -| `string` | msgpack str | -| `bin32`, `bytes` | msgpack bin | -| `vector` | msgpack bin (NOT array of integers) | -| `array` | msgpack array of integers | -| `vector` | msgpack array | -| `array` | msgpack array (fixed length) | -| `optional` | msgpack nil (if absent) or value | -| `alias` | same msgpack encoding as its primitive target | -| struct | msgpack map with string keys | -| NamedUnion | msgpack array(2): [string, map] | +msgpack uses the smallest encoding that fits the value, not the declared type: +a `u64` of `5` encodes as a single positive-fixint byte. Decoders MUST accept +any integer encoding width for any integer field. -### Integer Encoding Note +## Schema versioning -msgpack uses the **smallest encoding that fits the value**, not the declared type. -A `uint64_t` value of `5` encodes as a single byte (positive fixint), not as a -uint64 encoding. Decoders MUST accept any integer encoding width for any integer field. +A SHA-256 hash of the schema can be computed and embedded in generated code for +optional compatibility checking at connection time. A mismatch indicates the +service binary and client were generated from different schema versions. -## Schema Versioning +## Adding a new command -Schema compatibility can be validated by computing a SHA-256 hash of the raw JSON schema -output. This hash should be checked at connection time when possible. A mismatch indicates -that the service binary and client were generated from different schema versions. +1. Add an entry to `commands` with its `request`/`response` field maps (declare + any new `types`/`aliases` it needs). +2. Re-run ipc-codegen for every target language and confirm everything compiles. +3. If the change alters the wire format, refresh the golden corpus + (`./bootstrap.sh update_goldens`) and review the byte-level diff — any + change is breaking for external implementations of the schema. -## Adding a New Command +## Source files -To add a new command to a service: - -1. Define the command struct in C++ with `MSGPACK_SCHEMA_NAME` and `SERIALIZATION_FIELDS` -2. Add a nested `Response` struct with its own `MSGPACK_SCHEMA_NAME` and `SERIALIZATION_FIELDS` -3. Add both to the service's `Command` and `CommandResponse` NamedUnion types -4. Re-snapshot the schema JSON and re-run ipc-codegen for every target language -5. Verify generated code compiles in all target languages - -## Source Files - -- Schema visitor (IR compiler): `ipc-codegen/src/schema_visitor.ts` +- Schema front-end + IR compiler: `ipc-codegen/src/schema_visitor.ts` - CLI entry point: `ipc-codegen/src/generate.ts` - -The schema JSON is produced by the consumer's own C++ msgpack reflection (typically a ` msgpack schema` subcommand that walks `SERIALIZATION_FIELDS` and `NamedUnion`s and prints the IR). ipc-codegen treats the resulting JSON as the source of truth and never reaches back into the producer. +- Example schema: `ipc-codegen/echo_example/schema/schema.jsonc` diff --git a/ipc-codegen/bootstrap.sh b/ipc-codegen/bootstrap.sh index 841b171ef9c9..fa01a18fa960 100755 --- a/ipc-codegen/bootstrap.sh +++ b/ipc-codegen/bootstrap.sh @@ -55,7 +55,6 @@ function test_cmds { echo "$prefix $script golden ts" echo "$prefix $script golden cpp" echo "$prefix $script golden zig" - echo "$prefix ipc-codegen/echo_example/cpp/build/bin/schema_reflection_test --schema ipc-codegen/echo_example/schema/schema.json" echo "$prefix ipc-codegen/echo_example/ts_package/test.sh uds" echo "$prefix ipc-codegen/echo_example/ts_package/test.sh shm" diff --git a/ipc-codegen/echo_example/cpp/CMakeLists.txt b/ipc-codegen/echo_example/cpp/CMakeLists.txt index 52c5e51dc683..ab156f9b048a 100644 --- a/ipc-codegen/echo_example/cpp/CMakeLists.txt +++ b/ipc-codegen/echo_example/cpp/CMakeLists.txt @@ -48,8 +48,5 @@ add_executable(echo_client ) target_link_libraries(echo_client PRIVATE echo_common ipc_runtime) -add_executable(schema_reflection_test src/schema_reflection_test.cpp) -target_link_libraries(schema_reflection_test PRIVATE echo_common) - add_executable(golden_test src/golden_test.cpp) target_link_libraries(golden_test PRIVATE echo_common) diff --git a/ipc-codegen/echo_example/cpp/bootstrap.sh b/ipc-codegen/echo_example/cpp/bootstrap.sh index 38649124fac2..9f536ff54fe4 100755 --- a/ipc-codegen/echo_example/cpp/bootstrap.sh +++ b/ipc-codegen/echo_example/cpp/bootstrap.sh @@ -6,14 +6,12 @@ CODEGEN="$(cd "$DIR/../.." && pwd)" NODE="node --experimental-strip-types --experimental-transform-types --no-warnings" $NODE "$CODEGEN/src/generate.ts" \ - --schema "$DIR/../schema/schema.json" \ + --schema "$DIR/../schema/schema.jsonc" \ --lang cpp \ --server \ --client \ - --strip-method-prefix \ --out "$DIR/src/generated" \ - --prefix Echo \ --cpp-namespace echo cmake -S "$DIR" -B "$DIR/build" -cmake --build "$DIR/build" --target echo_server echo_client schema_reflection_test golden_test +cmake --build "$DIR/build" --target echo_server echo_client golden_test diff --git a/ipc-codegen/echo_example/cpp/src/schema_reflection_test.cpp b/ipc-codegen/echo_example/cpp/src/schema_reflection_test.cpp deleted file mode 100644 index 541421799eeb..000000000000 --- a/ipc-codegen/echo_example/cpp/src/schema_reflection_test.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Verifies the schema -> generated types -> reflected schema round trip is -// the identity. This is what makes the edit-code/extract-schema/commit -// workflow safe: reflecting the GENERATED wire types must reproduce the -// committed schema byte-for-byte (modulo whitespace). A hand-maintained copy -// of the types would mask generator drift (and did: it hid a union-ordering -// bug), so the generated header is reflected directly. - -#include "generated/echo_dispatch.hpp" -#include "generated/ipc_codegen/schema.hpp" - -#include -#include -#include -#include -#include - -namespace { - -std::string strip_whitespace(std::string value) { - std::string stripped; - stripped.reserve(value.size()); - for (unsigned char c : value) { - if (!std::isspace(c)) { - stripped.push_back(static_cast(c)); - } - } - return stripped; -} - -// Machinery self-check independent of codegen: a hand-declared struct must -// reflect to the expected JSON. -struct ReflectProbe { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "ReflectProbe"; - uint32_t value; - IPC_CODEGEN_SERIALIZATION_FIELDS(value) -}; - -bool machinery_self_check() { - auto reflected = ipc::msgpack_schema_to_string(ReflectProbe{}); - auto expected = R"({"__typename": "ReflectProbe", "value": "unsigned int"})"; - if (strip_whitespace(reflected) != strip_whitespace(expected)) { - std::cerr << "Reflection machinery self-check failed.\nGot: " << reflected - << "\nExpected: " << expected << "\n"; - return false; - } - return true; -} - -} // namespace - -int main(int argc, char **argv) { - if (argc != 3 || std::string(argv[1]) != "--schema") { - std::cerr << "Usage: schema_reflection_test --schema \n"; - return 1; - } - - if (!machinery_self_check()) { - return 1; - } - - std::ifstream schema_file(argv[2]); - if (!schema_file) { - std::cerr << "Failed to open schema: " << argv[2] << "\n"; - return 1; - } - std::stringstream buffer; - buffer << schema_file.rdbuf(); - - auto reflected = echo::get_echo_schema_as_json(); - if (strip_whitespace(reflected) != strip_whitespace(buffer.str())) { - std::cerr << "Reflected schema from GENERATED types does not match the " - "committed echo schema\n"; - std::cerr << "Reflected:\n" << reflected << "\n"; - return 1; - } - - std::cerr << "schema_reflection_test(cpp): generated-type roundtrip OK\n"; - return 0; -} diff --git a/ipc-codegen/echo_example/rust/bootstrap.sh b/ipc-codegen/echo_example/rust/bootstrap.sh index b95bae71dba0..570221c29f41 100755 --- a/ipc-codegen/echo_example/rust/bootstrap.sh +++ b/ipc-codegen/echo_example/rust/bootstrap.sh @@ -6,15 +6,13 @@ CODEGEN="$(cd "$DIR/../.." && pwd)" NODE="node --experimental-strip-types --experimental-transform-types --no-warnings" $NODE "$CODEGEN/src/generate.ts" \ - --schema "$DIR/../schema/schema.json" \ + --schema "$DIR/../schema/schema.jsonc" \ --lang rust \ --server \ --client \ - --strip-method-prefix \ --uds \ --ffi \ - --out "$DIR/src/generated" \ - --prefix Echo + --out "$DIR/src/generated" (cd "$DIR" && cargo build --locked --quiet) # Compile-check the generated FFI backend (not linked into the binaries). diff --git a/ipc-codegen/echo_example/schema/schema.json b/ipc-codegen/echo_example/schema/schema.json deleted file mode 100644 index cc9b676261e6..000000000000 --- a/ipc-codegen/echo_example/schema/schema.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "commands": [ - "named_union", - [ - [ - "EchoBytes", - { - "__typename": "EchoBytes", - "data": ["vector", ["unsigned char"]] - } - ], - [ - "EchoFields", - { - "__typename": "EchoFields", - "a": "unsigned int", - "b": "unsigned long", - "name": "string" - } - ], - [ - "EchoNested", - { - "__typename": "EchoNested", - "inner": { - "__typename": "EchoInner", - "values": ["vector", [["vector", ["unsigned char"]]]], - "flag": ["optional", ["bool"]] - } - } - ], - [ - "EchoAliases", - { - "__typename": "EchoAliases", - "treeId": "unsigned int", - "hash": ["alias", ["Fr", "bin32"]], - "maybeHash": ["optional", [["alias", ["Fr", "bin32"]]]], - "hashes": ["vector", [["alias", ["Fr", "bin32"]]]] - } - ], - [ - "EchoBlobs", - { - "__typename": "EchoBlobs", - "maybeData": ["optional", [["vector", ["unsigned char"]]]], - "parts": ["array", [["vector", ["unsigned char"]], 2]] - } - ], - [ - "EchoFail", - { - "__typename": "EchoFail", - "message": "string" - } - ] - ] - ], - "responses": [ - "named_union", - [ - [ - "EchoBytesResponse", - { - "__typename": "EchoBytesResponse", - "data": ["vector", ["unsigned char"]] - } - ], - [ - "EchoFieldsResponse", - { - "__typename": "EchoFieldsResponse", - "a": "unsigned int", - "b": "unsigned long", - "name": "string" - } - ], - [ - "EchoNestedResponse", - { - "__typename": "EchoNestedResponse", - "inner": "EchoInner" - } - ], - [ - "EchoAliasesResponse", - { - "__typename": "EchoAliasesResponse", - "treeId": "unsigned int", - "hash": ["alias", ["Fr", "bin32"]], - "maybeHash": ["optional", [["alias", ["Fr", "bin32"]]]], - "hashes": ["vector", [["alias", ["Fr", "bin32"]]]] - } - ], - [ - "EchoBlobsResponse", - { - "__typename": "EchoBlobsResponse", - "maybeData": ["optional", [["vector", ["unsigned char"]]]], - "parts": ["array", [["vector", ["unsigned char"]], 2]] - } - ], - [ - "EchoFailResponse", - { - "__typename": "EchoFailResponse" - } - ], - [ - "EchoErrorResponse", - { - "__typename": "EchoErrorResponse", - "message": "string" - } - ] - ] - ] -} diff --git a/ipc-codegen/echo_example/schema/schema.jsonc b/ipc-codegen/echo_example/schema/schema.jsonc new file mode 100644 index 000000000000..04c9a775dc85 --- /dev/null +++ b/ipc-codegen/echo_example/schema/schema.jsonc @@ -0,0 +1,44 @@ +// echo.schema.jsonc — the entire echo service, human-authored. +{ + "service": "Echo", + + // Named byte aliases (nominal 32-byte types). Only bin32 today. + "aliases": { + "Fr": "bin32" + }, + + // Shared struct types, referenced by name from commands. + "types": { + "EchoInner": { + "values": "bytes[]", + "flag": "bool?" + } + }, + + // Error variant shared by every command. + "error": { "message": "string" }, + + // command -> { request, response }. Names are unprefixed; generated type + // names get the service prefix (EchoBytes), method names do not (bytes). + "commands": { + "Bytes": { "request": { "data": "bytes" }, + "response": { "data": "bytes" } }, + + "Fields": { "request": { "a": "u32", "b": "u64", "name": "string" }, + "response": { "a": "u32", "b": "u64", "name": "string" } }, + + "Nested": { "request": { "inner": "EchoInner" }, + "response": { "inner": "EchoInner" } }, + + "Aliases": { "request": { "treeId": "u32", "hash": "Fr", + "maybeHash": "Fr?", "hashes": "Fr[]" }, + "response": { "treeId": "u32", "hash": "Fr", + "maybeHash": "Fr?", "hashes": "Fr[]" } }, + + "Blobs": { "request": { "maybeData": "bytes?", "parts": "bytes[2]" }, + "response": { "maybeData": "bytes?", "parts": "bytes[2]" } }, + + "Fail": { "request": { "message": "string" }, + "response": {} } + } +} diff --git a/ipc-codegen/echo_example/ts/bootstrap.sh b/ipc-codegen/echo_example/ts/bootstrap.sh index d783f705fb19..59c15eba21d2 100755 --- a/ipc-codegen/echo_example/ts/bootstrap.sh +++ b/ipc-codegen/echo_example/ts/bootstrap.sh @@ -7,12 +7,11 @@ REPO_ROOT="$(cd "$CODEGEN/.." && pwd)" NODE="node --experimental-strip-types --experimental-transform-types --no-warnings" $NODE "$CODEGEN/src/generate.ts" \ - --schema "$DIR/../schema/schema.json" \ + --schema "$DIR/../schema/schema.jsonc" \ --lang ts \ --server \ --client \ - --out "$DIR/src/generated" \ - --prefix Echo + --out "$DIR/src/generated" (cd "$REPO_ROOT/ipc-runtime" && ./bootstrap.sh) (cd "$REPO_ROOT/ipc-runtime/ts" && yarn install --immutable && yarn build) diff --git a/ipc-codegen/echo_example/ts/src/echo_client.ts b/ipc-codegen/echo_example/ts/src/echo_client.ts index 4009900b3cdb..81758913ce91 100644 --- a/ipc-codegen/echo_example/ts/src/echo_client.ts +++ b/ipc-codegen/echo_example/ts/src/echo_client.ts @@ -58,12 +58,12 @@ async function run() { // Test 1: EchoBytes const testData = Uint8Array.from([0xde, 0xad, 0xbe, 0xef, 0x42]); - const resp1 = await api.echoBytes({ data: testData }); + const resp1 = await api.bytes({ data: testData }); assertBytes(resp1.data, testData, "EchoBytes data"); console.error("echo_client(ts): EchoBytes OK"); // Test 2: EchoFields - const resp2 = await api.echoFields({ + const resp2 = await api.fields({ a: 42, b: 999999, name: "hello wire compat", @@ -78,7 +78,7 @@ async function run() { values: [Uint8Array.from([1, 2, 3]), Uint8Array.from([4, 5])], flag: true, }; - const resp3 = await api.echoNested({ inner }); + const resp3 = await api.nested({ inner }); assertEqual(resp3.inner.flag, true, "EchoNested flag"); assertEqual(resp3.inner.values.length, 2, "EchoNested values length"); assertBytes(resp3.inner.values[0]!, inner.values[0]!, "EchoNested values[0]"); @@ -87,7 +87,7 @@ async function run() { // Test 4: EchoAliases const hash = testHash(0x10); const second = testHash(0x40); - const resp4 = await api.echoAliases({ + const resp4 = await api.aliases({ treeId: 7, hash, maybeHash: second, @@ -102,7 +102,7 @@ async function run() { console.error("echo_client(ts): EchoAliases OK"); // Test 5: EchoAliases with maybeHash absent (optional over live IPC) - const resp5 = await api.echoAliases({ + const resp5 = await api.aliases({ treeId: 7, hash, maybeHash: null, @@ -113,12 +113,12 @@ async function run() { // Test 6: EchoFields with b > 2^32 (uint64 wire encoding over live IPC) const big = Number.MAX_SAFE_INTEGER; - const resp6 = await api.echoFields({ a: 42, b: big, name: "big" }); + const resp6 = await api.fields({ a: 42, b: big, name: "big" }); assertEqual(resp6.b, big, "EchoFields u64"); // Values past 2^53 must throw client-side rather than silently lose precision. let threw = false; try { - await api.echoFields({ a: 42, b: 2 ** 60, name: "too big" }); + await api.fields({ a: 42, b: 2 ** 60, name: "too big" }); } catch { threw = true; } @@ -126,7 +126,7 @@ async function run() { console.error("echo_client(ts): EchoFields u64 OK"); // Test 7: EchoBlobs — optional bytes Some/None and fixed [bytes; 2] - const resp7 = await api.echoBlobs({ + const resp7 = await api.blobs({ maybeData: Uint8Array.from([0xaa, 0xbb]), parts: [Uint8Array.from([1, 2, 3]), Uint8Array.from([4])], }); @@ -141,7 +141,7 @@ async function run() { "EchoBlobs parts[0]", ); assertBytes(resp7.parts[1]!, Uint8Array.from([4]), "EchoBlobs parts[1]"); - const resp7b = await api.echoBlobs({ + const resp7b = await api.blobs({ maybeData: null, parts: [Uint8Array.from([]), Uint8Array.from([9])], }); @@ -151,7 +151,7 @@ async function run() { // Test 8: EchoFail — server error surfaces with its message let failMessage = ""; try { - await api.echoFail({ message: "deliberate failure" }); + await api.fail({ message: "deliberate failure" }); } catch (e: any) { failMessage = e.message; } diff --git a/ipc-codegen/echo_example/ts/src/echo_server.ts b/ipc-codegen/echo_example/ts/src/echo_server.ts index 5a992e5685b4..1951a296b67c 100644 --- a/ipc-codegen/echo_example/ts/src/echo_server.ts +++ b/ipc-codegen/echo_example/ts/src/echo_server.ts @@ -30,16 +30,16 @@ if (!socketPath) { } const handler: Handler = { - async echoBytes(cmd: EchoBytes): Promise { + async bytes(cmd: EchoBytes): Promise { return { data: cmd.data }; }, - async echoFields(cmd: EchoFields): Promise { + async fields(cmd: EchoFields): Promise { return { a: cmd.a, b: cmd.b, name: cmd.name }; }, - async echoNested(cmd: EchoNested): Promise { + async nested(cmd: EchoNested): Promise { return { inner: cmd.inner }; }, - async echoAliases(cmd: EchoAliases): Promise { + async aliases(cmd: EchoAliases): Promise { return { treeId: cmd.treeId, hash: cmd.hash, @@ -47,10 +47,10 @@ const handler: Handler = { hashes: cmd.hashes, }; }, - async echoBlobs(cmd: EchoBlobs): Promise { + async blobs(cmd: EchoBlobs): Promise { return { maybeData: cmd.maybeData, parts: cmd.parts }; }, - async echoFail(cmd: EchoFail): Promise { + async fail(cmd: EchoFail): Promise { throw new Error(cmd.message); }, }; diff --git a/ipc-codegen/echo_example/ts_package/bootstrap.sh b/ipc-codegen/echo_example/ts_package/bootstrap.sh index 89004eab4a70..7b4405b3d2ad 100755 --- a/ipc-codegen/echo_example/ts_package/bootstrap.sh +++ b/ipc-codegen/echo_example/ts_package/bootstrap.sh @@ -7,12 +7,10 @@ REPO_ROOT="$(cd "$CODEGEN/.." && pwd)" NODE="node --experimental-strip-types --experimental-transform-types --no-warnings" $NODE "$CODEGEN/src/generate.ts" \ - --schema "$DIR/../schema/schema.json" \ + --schema "$DIR/../schema/schema.jsonc" \ --lang ts \ --client \ --out "$DIR/src/generated" \ - --prefix Echo \ - --strip-method-prefix \ --package "$DIR" \ --package-name "@aztec/echo-ipc" \ --binary-name echo_server \ diff --git a/ipc-codegen/echo_example/zig/bootstrap.sh b/ipc-codegen/echo_example/zig/bootstrap.sh index 90484ee376b9..a65642aac36a 100755 --- a/ipc-codegen/echo_example/zig/bootstrap.sh +++ b/ipc-codegen/echo_example/zig/bootstrap.sh @@ -6,14 +6,12 @@ CODEGEN="$(cd "$DIR/../.." && pwd)" NODE="node --experimental-strip-types --experimental-transform-types --no-warnings" $NODE "$CODEGEN/src/generate.ts" \ - --schema "$DIR/../schema/schema.json" \ + --schema "$DIR/../schema/schema.jsonc" \ --lang zig \ --server \ --client \ - --strip-method-prefix \ --uds \ --ffi \ - --out "$DIR/src/generated" \ - --prefix Echo + --out "$DIR/src/generated" (cd "$DIR" && zig build) diff --git a/ipc-codegen/scripts/convert_schema.ts b/ipc-codegen/scripts/convert_schema.ts new file mode 100644 index 000000000000..0f0bed844be8 --- /dev/null +++ b/ipc-codegen/scripts/convert_schema.ts @@ -0,0 +1,173 @@ +// One-shot converter: old positional IPC schema -> friendly JSONC form. +// Usage: node convert_schema.ts [outFriendly.jsonc] +import { readFileSync, writeFileSync } from "fs"; +import { + SchemaVisitor, + friendlyToPositional, +} from "../src/schema_visitor.ts"; + +const PRIM: Record = { + bool: "bool", + int: "u32", + "unsigned int": "u32", + "unsigned short": "u16", + "unsigned long": "u64", + "unsigned long long": "u64", + "unsigned char": "u8", + double: "f64", + string: "string", + bin32: "bin32", +}; + +function convert(oldPath: string, prefix: string) { + const old = JSON.parse(readFileSync(oldPath, "utf-8")); + const aliases: Record = {}; + const types: Record> = {}; + + const typeToShorthand = (t: any): string => { + if (typeof t === "string") { + return PRIM[t] ?? t; // primitive or a named (struct) reference + } + if (Array.isArray(t)) { + const [kind, args] = t; + if (kind === "vector") { + const [el] = args; + if (el === "unsigned char") return "bytes"; + return typeToShorthand(el) + "[]"; + } + if (kind === "array") + return typeToShorthand(args[0]) + "[" + args[1] + "]"; + if (kind === "optional") return typeToShorthand(args[0]) + "?"; + if (kind === "shared_ptr") return typeToShorthand(args[0]); + if (kind === "alias") { + const [name, underlying] = args; + aliases[name] = underlying === "bin32" ? "bin32" : PRIM[underlying]; + if (!aliases[name]) + throw new Error(`alias ${name} underlying ${underlying}`); + return name; + } + throw new Error(`unknown type kind: ${kind}`); + } + if (t && typeof t === "object" && t.__typename) { + const tn = t.__typename as string; + if (!(tn in types)) { + types[tn] = {}; // reserve to break cycles + types[tn] = structFields(t); + } + return tn; + } + throw new Error(`cannot convert type ${JSON.stringify(t)}`); + }; + + const structFields = (struct: any): Record => { + const out: Record = {}; + for (const [k, v] of Object.entries(struct)) { + if (k === "__typename") continue; + out[k] = typeToShorthand(v); + } + return out; + }; + + const commandPairs = old.commands[1] as Array<[string, any]>; + const responsePairs = old.responses[1] as Array<[string, any]>; + const respByName = new Map(responsePairs); + + const errEntry = responsePairs.find(([n]) => n.endsWith("ErrorResponse"))!; + const error = structFields(errEntry[1]); + + const commands: Record = {}; + for (const [cmdName, cmdStruct] of commandPairs) { + const key = cmdName.startsWith(prefix) + ? cmdName.slice(prefix.length) + : cmdName; + const request = structFields(cmdStruct); + const respStruct = respByName.get(`${cmdName}Response`); + let response: any; + if (respStruct === undefined) { + throw new Error(`No response named ${cmdName}Response`); + } else if (typeof respStruct === "string") { + response = respStruct.startsWith(prefix) + ? respStruct.slice(prefix.length) + : respStruct; + } else { + response = structFields(respStruct); + } + commands[key] = { request, response }; + } + + return { service: prefix, aliases, types, error, commands }; +} + +// Deep structural equality of CompiledSchema, ignoring Map insertion order. +// Struct references are compared by NAME only: generators emit nested structs +// by name from the top-level `structs` map, so the embedded fields on a struct +// ref are irrelevant (and differ benignly between string-ref and inline forms). +function normalize(c: any): any { + const normType = (t: any): any => { + if (t == null || typeof t !== "object") return t; + if (t.kind === "struct") + return { kind: "struct", structName: t.struct?.name }; + if (t.element) + return { kind: t.kind, size: t.size, element: normType(t.element) }; + return { + kind: t.kind, + primitive: t.primitive, + originalName: t.originalName, + }; + }; + const norm = (s: any) => ({ + name: s.name, + fields: s.fields.map((f: any) => ({ + name: f.name, + type: normType(f.type), + })), + }); + return { + structs: Object.fromEntries( + [...c.structs.entries()].map(([k, v]: any) => [k, norm(v)]).sort(), + ), + responses: Object.fromEntries( + [...c.responses.entries()].map(([k, v]: any) => [k, norm(v)]).sort(), + ), + commands: [...c.commands] + .map((x: any) => ({ + name: x.name, + responseType: x.responseType, + fields: x.fields.map((f: any) => ({ + name: f.name, + type: normType(f.type), + })), + })) + .sort((a, b) => a.name.localeCompare(b.name)), + errorTypeName: c.errorTypeName, + }; +} + +const [oldPath, prefix, outPath] = process.argv.slice(2); +const friendly = convert(oldPath, prefix); +const friendlyText = JSON.stringify(friendly, null, 2); +if (outPath) writeFileSync(outPath, friendlyText + "\n"); + +const old = JSON.parse(readFileSync(oldPath, "utf-8")); +const oldCompiled = new SchemaVisitor().visit(old.commands, old.responses); +const { commands, responses } = friendlyToPositional(friendly); +const newCompiled = new SchemaVisitor().visit(commands, responses); + +const a = JSON.stringify(normalize(oldCompiled)); +const b = JSON.stringify(normalize(newCompiled)); +if (a === b) { + console.log( + `ROUND-TRIP OK service=${prefix} commands=${friendly.commands && Object.keys(friendly.commands).length} types=${Object.keys(friendly.types).length} aliases=${Object.keys(friendly.aliases).length}`, + ); +} else { + console.log(`ROUND-TRIP MISMATCH for ${prefix}`); + // show first divergence + for (let i = 0; i < Math.max(a.length, b.length); i++) { + if (a[i] !== b[i]) { + console.log("old:", a.slice(Math.max(0, i - 80), i + 80)); + console.log("new:", b.slice(Math.max(0, i - 80), i + 80)); + break; + } + } + process.exit(1); +} diff --git a/ipc-codegen/src/cpp_codegen.ts b/ipc-codegen/src/cpp_codegen.ts index 2403e7671a97..d247220cb278 100644 --- a/ipc-codegen/src/cpp_codegen.ts +++ b/ipc-codegen/src/cpp_codegen.ts @@ -388,10 +388,9 @@ ${methods} .sort(([a], [b]) => a.localeCompare(b)) .map(([name, { underlying, schemaName }]) => { // bin32 aliases are nominal types (a fixed 32-byte value with a name), - // so they reflect their alias name. Scalar aliases are transparent + // so they are distinct wrapper structs. Scalar aliases are transparent // synonyms — consumers static_cast them to/from enums and integers — - // so they are plain `using` and do not carry their name through - // reflection. + // so they are plain `using`. if (underlying === "std::array") { return `struct ${name} : ::ipc::Bin32Alias<${name}> { using ::ipc::Bin32Alias<${name}>::Bin32Alias; @@ -466,7 +465,6 @@ ${methods} // --------------------------------------------------------------------------- // Self-contained serialization macro for generated wire types. // Defines a msgpack() method that enumerates field name/value pairs. -// Works with msgpack packers (serialization) and schema reflectors. // --------------------------------------------------------------------------- #ifndef IPC_CODEGEN_SERIALIZATION_FIELDS #define _SF_E1(x) #x, x @@ -500,7 +498,7 @@ ${methods} // --------------------------------------------------------------------------- // Wire aliases for primitive schema aliases. bin32 aliases are nominal wrappers -// so schema reflection can preserve their alias names. +// carrying their alias name as the MSGPACK_SCHEMA_NAME dispatch tag. // --------------------------------------------------------------------------- #ifndef IPC_CODEGEN_BIN32_ALIAS_DEFINED @@ -544,7 +542,6 @@ template struct Bin32Alias { } } - void msgpack_schema(auto& packer) const { packer.pack_alias(Tag::MSGPACK_SCHEMA_NAME, "bin32"); } bool operator==(const Bin32Alias&) const = default; }; } // namespace ipc @@ -572,22 +569,6 @@ ${this.opts.wireNamespace ? `} // namespace ${this.opts.wireNamespace}` : ""} const { namespace: ns, prefix } = this.opts; const errorTypeName = schema.errorTypeName; const typesHeader = `${toSnakeCase(prefix)}_types.hpp`; - const prefixLower = toSnakeCase(prefix); - - // Per-service NamedUnions + schema reflection. The codegen-emitted - // Command / CommandResponse aggregate every wire type - // so the binary can pack its own schema back out via - // ipc::msgpack_schema_to_string. This is the C++-canonical dev workflow: - // edit a wire type, rebuild, dump the schema, commit the JSON. - const cmdUnionMembers = schema.commands - .map((c) => `wire::${c.name}`) - .join(",\n "); - // Union members must be emitted in schema order so that reflecting the - // generated types reproduces the committed schema byte-for-byte. - const respUnionMembers = [...schema.responses.keys()] - .map((r) => `wire::${r}`) - .join(",\n "); - // Handler declarations — template const handlerDecls = schema.commands .map((c) => { @@ -635,8 +616,6 @@ ${this.opts.wireNamespace ? `} // namespace ${this.opts.wireNamespace}` : ""} #pragma once #include "${typesHeader}" -#include "ipc_codegen/named_union.hpp" -#include "ipc_codegen/schema.hpp" #include "ipc_codegen/msgpack_adaptor.hpp" // Pull in THROW/RETHROW — 'throw' natively, abort-on-throw under @@ -724,34 +703,6 @@ ${handlerEntries}, }; } -// --------------------------------------------------------------------------- -// Schema reflection — the binary serialises its own understanding of the wire -// format. Edit a wire type, rebuild, dump the schema, commit the JSON. -// --------------------------------------------------------------------------- - -using ${prefix}Command = ::ipc::NamedUnion<${cmdUnionMembers}>; -using ${prefix}CommandResponse = ::ipc::NamedUnion<${respUnionMembers}>; - -namespace detail { -// Reflects as the bare {"commands": ..., "responses": ...} document so the -// output is exactly the committable schema (no wrapper __typename). -struct ${prefix}Api { - void msgpack_schema(auto& packer) const - { - packer.pack_map(2); - packer.pack("commands"); - packer.pack_schema(${prefix}Command{}); - packer.pack("responses"); - packer.pack_schema(${prefix}CommandResponse{}); - } -}; -} // namespace detail - -inline std::string get_${prefixLower}_schema_as_json() -{ - return ::ipc::msgpack_schema_to_string(detail::${prefix}Api{}); -} - } // namespace ${ns} `; } diff --git a/ipc-codegen/src/generate.ts b/ipc-codegen/src/generate.ts index 3808e69baa1d..79ef1a15e1b6 100644 --- a/ipc-codegen/src/generate.ts +++ b/ipc-codegen/src/generate.ts @@ -27,7 +27,13 @@ import { import { execSync } from "child_process"; import { dirname, join, resolve } from "path"; import { fileURLToPath } from "url"; -import { SchemaVisitor, type CompiledSchema } from "./schema_visitor.ts"; +import { + SchemaVisitor, + friendlyToPositional, + isFriendlySchema, + stripJsonc, + type CompiledSchema, +} from "./schema_visitor.ts"; import { TypeScriptCodegen } from "./typescript_codegen.ts"; import { defaultBinaryEnvVar, @@ -230,13 +236,27 @@ function computeSchemaHash(schemaJson: string): string { function loadSchema(schemaPath: string): { compiled: CompiledSchema; schemaHash: string; + service?: string; } { const rawJson = readFileSync(schemaPath, "utf-8").trim(); - const schema = JSON.parse(rawJson); + const parsed = JSON.parse(stripJsonc(rawJson)); + let commandsUnion: any; + let responsesUnion: any; + let service: string | undefined; + if (isFriendlySchema(parsed)) { + ({ + commands: commandsUnion, + responses: responsesUnion, + service, + } = friendlyToPositional(parsed)); + } else { + commandsUnion = parsed.commands; + responsesUnion = parsed.responses; + } const visitor = new SchemaVisitor(); - const compiled = visitor.visit(schema.commands, schema.responses); + const compiled = visitor.visit(commandsUnion, responsesUnion); const schemaHash = computeSchemaHash(rawJson); - return { compiled, schemaHash }; + return { compiled, schemaHash, service }; } /** Detect common prefix from command names (e.g. WsdbGetTreeInfo, WsdbCreateFork → Wsdb) */ @@ -308,8 +328,12 @@ function generate(args: Args) { const absOut = resolve(args.out); mkdirSync(absOut, { recursive: true }); - const { compiled, schemaHash } = loadSchema(absSchema); - const prefix = args.prefix || detectPrefix(compiled); + const { compiled, schemaHash, service } = loadSchema(absSchema); + // Friendly schemas fold the type prefix and method-prefix stripping into + // `service`: generated type names are `service + command`, method names are + // the bare command. Positional schemas keep the legacy --prefix/--strip flags. + const prefix = service || args.prefix || detectPrefix(compiled); + const stripMethodPrefix = service ? true : args.stripMethodPrefix; console.log( `Schema: ${absSchema} (${compiled.commands.length} commands, prefix=${prefix})`, @@ -334,7 +358,7 @@ function generate(args: Args) { switch (args.lang) { case "ts": { const gen = new TypeScriptCodegen({ - stripMethodPrefix: args.stripMethodPrefix ? prefix : undefined, + stripMethodPrefix: stripMethodPrefix ? prefix : undefined, }); writeFile("api_types.ts", gen.generateTypes(compiled, schemaHash)); if (args.server) { @@ -406,7 +430,10 @@ function generate(args: Args) { break; } case "rust": { - const gen = new RustCodegen({ prefix, stripMethodPrefix: args.stripMethodPrefix }); + const gen = new RustCodegen({ + prefix, + stripMethodPrefix: stripMethodPrefix, + }); writeFile( `${toSnakeCase(prefix)}_types.rs`, gen.generateTypes(compiled, schemaHash), @@ -436,7 +463,11 @@ function generate(args: Args) { break; } case "zig": { - const gen = new ZigCodegen({ prefix, clientName: `${prefix}Client`, stripMethodPrefix: args.stripMethodPrefix }); + const gen = new ZigCodegen({ + prefix, + clientName: `${prefix}Client`, + stripMethodPrefix: stripMethodPrefix, + }); writeFile( `${toSnakeCase(prefix)}_types.zig`, gen.generateTypes(compiled, schemaHash), @@ -475,7 +506,7 @@ function generate(args: Args) { prefix, wireNamespace: wireNs, generatedIncludeDir: args.cppIncludeDir, - stripMethodPrefix: args.stripMethodPrefix, + stripMethodPrefix: stripMethodPrefix, }); cppFiles.push( @@ -514,7 +545,6 @@ function generate(args: Args) { ); } - formatCpp(cppFiles); break; } diff --git a/ipc-codegen/src/rust_codegen.ts b/ipc-codegen/src/rust_codegen.ts index 660dd7a21b20..d0b34c2ce4b8 100644 --- a/ipc-codegen/src/rust_codegen.ts +++ b/ipc-codegen/src/rust_codegen.ts @@ -178,12 +178,6 @@ export class RustCodegen { const serdeRename = struct.name !== rustName ? `\n#[serde(rename = "${struct.name}")]` : ""; - // Commands have a __typename used for NamedUnion identification, but it's handled - // by the Command enum's custom serde, not by the struct itself. - const typenameField = isCommand - ? ` #[serde(rename = "__typename", skip, default)]\n pub type_name: String,\n` - : ""; - // Generate constructor for commands const constructor = isCommand ? this.generateConstructor(struct, rustName) @@ -192,7 +186,7 @@ export class RustCodegen { return `/// ${struct.name} #[derive(Debug, Clone, Serialize, Deserialize)]${serdeRename} pub struct ${rustName} { -${typenameField}${fields} +${fields} }${constructor}`; } @@ -202,10 +196,9 @@ ${typenameField}${fields} .map((f) => `${toSnakeCase(f.name)}: ${this.mapType(f.type)}`) .join(", "); - const fieldInits = [ - ` type_name: "${struct.name}".to_string(),`, - ...struct.fields.map((f) => ` ${toSnakeCase(f.name)},`), - ].join("\n"); + const fieldInits = struct.fields + .map((f) => ` ${toSnakeCase(f.name)},`) + .join("\n"); return ` @@ -487,7 +480,8 @@ mod serde_opt_bytes { // Generate types file generateTypes(schema: CompiledSchema, schemaHash?: string): string { this.errorTypeName = schema.errorTypeName; - // Create set of top-level command struct names (only these need __typename) + // Command structs get a generated `new()` constructor; response/shared + // structs do not. const commandNames = new Set(schema.commands.map((c) => c.name)); const aliasTypes = new Map(); diff --git a/ipc-codegen/src/schema_visitor.ts b/ipc-codegen/src/schema_visitor.ts index aa8d151f0b3f..8e98ea9d5424 100644 --- a/ipc-codegen/src/schema_visitor.ts +++ b/ipc-codegen/src/schema_visitor.ts @@ -506,3 +506,173 @@ export class SchemaVisitor { } } } + +// --------------------------------------------------------------------------- +// Friendly (human-authored) schema front-end +// +// The committed schemas are hand-edited. The friendly format is a single +// object per service with shorthand string type references; this front-end +// lowers it to the positional ["named_union", ...] form that SchemaVisitor +// already consumes, so the generators are untouched and the produced +// CompiledSchema is identical to the equivalent positional schema. +// --------------------------------------------------------------------------- + +/** Strip line and block comments from JSONC, preserving string contents. */ +export function stripJsonc(text: string): string { + let out = ""; + let inStr = false; + let strCh = ""; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + const n = text[i + 1]; + if (inStr) { + out += c; + if (c === "\\") { + out += n ?? ""; + i++; + } else if (c === strCh) { + inStr = false; + } + continue; + } + if (c === '"' || c === "'") { + inStr = true; + strCh = c; + out += c; + continue; + } + if (c === "/" && n === "/") { + while (i < text.length && text[i] !== "\n") i++; + continue; + } + if (c === "/" && n === "*") { + i += 2; + while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) i++; + i++; // skip the closing '/' + continue; + } + out += c; + } + return out; +} + +/** A parsed friendly schema is recognised by its top-level `service` key. */ +export function isFriendlySchema(parsed: any): boolean { + return ( + parsed != null && + typeof parsed === "object" && + !Array.isArray(parsed) && + typeof parsed.service === "string" + ); +} + +// u32 etc. -> the positional primitive spellings resolvePrimitive() accepts. +const PRIMITIVE_SHORTHAND: Record = { + bool: "bool", + u8: "unsigned char", + u16: "unsigned short", + u32: "unsigned int", + u64: "unsigned long", + f64: "double", + string: "string", + bin32: "bin32", +}; + +/** + * Lower a friendly schema object into the positional `{ commands, responses }` + * named_union pair, plus the service name (used as the type prefix). Type names + * are `service + commandKey`; method names are derived by the generators via + * the service prefix. Named `types` are inlined at every reference — visit() + * dedups them by `__typename`, so the resulting CompiledSchema matches the + * positional form exactly. + */ +export function friendlyToPositional(parsed: any): { + commands: any; + responses: any; + service: string; +} { + const service: string = parsed.service; + if (!service) { + throw new Error("Friendly schema requires a non-empty string 'service'"); + } + const aliases: Record = parsed.aliases ?? {}; + const types: Record = parsed.types ?? {}; + // An alias is either a nominal byte type (bin32) or a transparent scalar + // synonym over a primitive (e.g. MerkleTreeId = u32). The underlying is given + // in shorthand; map it to the positional spelling visitType() consumes. + const aliasUnderlying = (name: string): string => { + const u = aliases[name]; + if (u === "bin32") return "bin32"; + const prim = PRIMITIVE_SHORTHAND[u]; + if (!prim) { + throw new Error( + `Alias '${name}' underlying '${u}' is not 'bin32' or a primitive`, + ); + } + return prim; + }; + for (const [name] of Object.entries(aliases)) { + aliasUnderlying(name); // validate up front + } + + const parseTypeRef = (ref: string): any => { + const s = ref.trim(); + if (s.endsWith("?")) { + return ["optional", [parseTypeRef(s.slice(0, -1))]]; + } + if (s.endsWith("]")) { + const lb = s.lastIndexOf("["); + if (lb < 0) throw new Error(`Malformed type reference '${ref}'`); + const inner = s.slice(0, lb); + const n = s.slice(lb + 1, -1).trim(); + if (n === "") { + return ["vector", [parseTypeRef(inner)]]; + } + const size = Number(n); + if (!Number.isInteger(size) || size <= 0) { + throw new Error(`Bad fixed-array size in type reference '${ref}'`); + } + return ["array", [parseTypeRef(inner), size]]; + } + if (s === "bytes") return ["vector", ["unsigned char"]]; + if (s in PRIMITIVE_SHORTHAND) return PRIMITIVE_SHORTHAND[s]; + if (s in aliases) return ["alias", [s, aliasUnderlying(s)]]; + if (s in types) return inlineType(s); + throw new Error( + `Unknown type reference '${ref}' (not a primitive, declared alias, or declared type)`, + ); + }; + + const structBody = (fieldObj: Record, typename: string) => { + const body: any = { __typename: typename }; + for (const [fname, fref] of Object.entries(fieldObj ?? {})) { + body[fname] = parseTypeRef(fref); + } + return body; + }; + + const inlineType = (typeName: string): any => + structBody(types[typeName], typeName); + + const commands: any = ["named_union", []]; + const responses: any = ["named_union", []]; + + for (const [key, def] of Object.entries(parsed.commands ?? {})) { + const cmdName = service + key; + commands[1].push([cmdName, structBody(def.request, cmdName)]); + + if (typeof def.response === "string") { + // Reuse another command's response shape: dedup string-ref form. + const respName = service + def.response; + responses[1].push([respName, respName]); + } else { + const respName = `${cmdName}Response`; + responses[1].push([respName, structBody(def.response, respName)]); + } + } + + const errorName = `${service}ErrorResponse`; + responses[1].push([errorName, structBody(parsed.error, errorName)]); + + return { commands, responses, service }; +} diff --git a/ipc-codegen/templates/cpp/ipc_codegen/named_union.hpp b/ipc-codegen/templates/cpp/ipc_codegen/named_union.hpp index 0d7df4cbd9ba..f340017ff4c8 100644 --- a/ipc-codegen/templates/cpp/ipc_codegen/named_union.hpp +++ b/ipc-codegen/templates/cpp/ipc_codegen/named_union.hpp @@ -1,16 +1,16 @@ #pragma once /** * @file named_union.hpp - * @brief Tagged-union with msgpack [name, payload] wire format. Single source - * of truth used by codegen-emitted dispatchers and schema reflection. + * @brief Tagged-union with msgpack [name, payload] wire format, used by + * codegen-emitted dispatchers. * * Each type in the union must declare: * static constexpr const char MSGPACK_SCHEMA_NAME[] = "..."; */ #include "throw.hpp" -#include #include "msgpack_include.hpp" +#include #include #include #include @@ -112,21 +112,6 @@ template class NamedUnion { } value_ = construct_by_index(*index_opt, arr.ptr[1]); } - - // Schema reflection — emits ["named_union", [[name, schema], ...]] via - // the schema packer (see reflect.hpp). - void msgpack_schema(auto &packer) const { - packer.pack_array(2); - packer.pack("named_union"); - packer.pack_array(sizeof...(Types)); - ( - [&packer]() { - packer.pack_array(2); - packer.pack(Types::MSGPACK_SCHEMA_NAME); - packer.pack_schema(*std::make_unique()); - }(), - ...); - } }; } // namespace ipc diff --git a/ipc-codegen/templates/cpp/ipc_codegen/schema.hpp b/ipc-codegen/templates/cpp/ipc_codegen/schema.hpp deleted file mode 100644 index 07c196204f00..000000000000 --- a/ipc-codegen/templates/cpp/ipc_codegen/schema.hpp +++ /dev/null @@ -1,214 +0,0 @@ -#pragma once -/** - * @file schema.hpp - * @brief Compile-time msgpack schema reflection for codegen-emitted types. - * - * Walks a type's `msgpack(pack_fn)` method (which SERIALIZATION_FIELDS or the - * codegen-emitted bundled adaptor provides) and produces a JSON description - * of its msgpack layout. The output format is consumed by ipc-codegen as the - * canonical schema source — the binary serialises its own understanding of - * the wire format and that becomes the input for cross-language codegen. - * - * The schema reflection itself is in this file (stdlib + msgpack-c only) so - * services consuming ipc-codegen output do not need project-specific headers. - */ -#include -#include -#include -#include -#include -#include "msgpack_include.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ipc { - -// ---------------------------------------------------------------------------- -// Type names -// ---------------------------------------------------------------------------- - -template std::string schema_name(T const &) { - if constexpr (requires { T::MSGPACK_SCHEMA_NAME; }) { - return T::MSGPACK_SCHEMA_NAME; - } else { - char *demangled = - abi::__cxa_demangle(typeid(T).name(), nullptr, nullptr, nullptr); - std::string result = demangled ? demangled : typeid(T).name(); - if (demangled) - std::free(demangled); // NOLINT - // basic_string<...> → "string" - if (result.find("basic_string") != std::string::npos) - return "string"; - if (result == "i") - return "int"; - // Strip template args (Foo<...> → Foo) - if (auto pos = result.find('<'); pos != std::string::npos) - result = result.substr(0, pos); - // Strip namespace prefix (a::b::c → c) - if (auto pos = result.rfind(':'); pos != std::string::npos) - result = result.substr(pos + 1); - return result; - } -} - -// ---------------------------------------------------------------------------- -// Concepts -// ---------------------------------------------------------------------------- - -namespace schema_detail { -struct DoNothing { - void operator()(auto...) {} -}; -template -concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); }; -template -concept HasMsgPackSchema = - requires(const T t, DoNothing nop) { t.msgpack_schema(nop); }; -} // namespace schema_detail - -// ---------------------------------------------------------------------------- -// Schema packer -// ---------------------------------------------------------------------------- - -struct SchemaPacker; - -template -inline void schema_pack(SchemaPacker &packer, T const &obj); - -struct SchemaPacker : msgpack::packer { - SchemaPacker(msgpack::sbuffer &stream) : packer(stream) {} - - std::set emitted_types; - bool set_emitted(const std::string &type) { - if (emitted_types.find(type) == emitted_types.end()) { - emitted_types.insert(type); - return false; - } - return true; - } - - template void pack_schema(T const &obj) { - schema_pack(*this, obj); - } - - template void pack_template_type(const std::string &name) { - pack_array(2); - pack(name); - pack_array(sizeof...(Args)); - (schema_pack(*this, *std::make_unique()), ...); - } - - // ["alias", [, ]] — preserves the alias name in - // the emitted schema while pinning the underlying msgpack type. - void pack_alias(const std::string &schema_name, - const std::string &msgpack_name) { - pack_array(2); - pack("alias"); - pack_array(2); - pack(schema_name); - pack(msgpack_name); - } - - template - void pack_with_name(const std::string &type, T const &object) { - if (set_emitted(type)) { - pack(type); - return; - } - const_cast(object).msgpack([&](auto &...args) { - size_t kv_size = sizeof...(args); - pack_map(uint32_t(1 + kv_size / 2)); - pack("__typename"); - pack(type); - _schema_pack_map_content(*this, args...); - }); - } -}; - -inline void _schema_pack_map_content(SchemaPacker &) {} - -template -inline void _schema_pack_map_content(SchemaPacker &packer, std::string key, - const Value &value, const Rest &...rest) { - packer.pack(key); - schema_pack(packer, value); - _schema_pack_map_content(packer, rest...); -} - -// Fallback for types with no msgpack method (primitives, etc.) -template - requires(!schema_detail::HasMsgPackSchema && !schema_detail::HasMsgPack) -inline void schema_pack(SchemaPacker &packer, T const &obj) { - packer.pack(schema_name(obj)); -} - -// Type with custom msgpack_schema method (e.g. NamedUnion) -template -inline void schema_pack(SchemaPacker &packer, T const &obj) { - obj.msgpack_schema(packer); -} - -// Type with SERIALIZATION_FIELDS — pack as a map -template - requires(!schema_detail::HasMsgPackSchema) -inline void schema_pack(SchemaPacker &packer, T const &object) { - packer.pack_with_name(schema_name(object), object); -} - -// Container overloads -template -inline void schema_pack(SchemaPacker &packer, std::vector const &) { - packer.pack_template_type("vector"); -} -template -inline void schema_pack(SchemaPacker &packer, std::optional const &) { - packer.pack_template_type("optional"); -} -template -inline void schema_pack(SchemaPacker &packer, std::tuple const &) { - packer.pack_template_type("tuple"); -} -template -inline void schema_pack(SchemaPacker &packer, std::map const &) { - packer.pack_template_type("map"); -} -template -inline void schema_pack(SchemaPacker &packer, std::variant const &) { - packer.pack_template_type("variant"); -} -template -inline void schema_pack(SchemaPacker &packer, std::array const &) { - // Exactly 32 bytes is the fixed-byte primitive used by bin32 aliases. - if constexpr (N == 32 && (std::is_same_v || - std::is_same_v)) { - packer.pack("bin32"); - } else { - packer.pack_array(2); - packer.pack("array"); - packer.pack_array(2); - schema_pack(packer, *std::make_unique()); - packer.pack(N); - } -} - -// ---------------------------------------------------------------------------- -// Convenience: serialise an object's schema to a JSON-ish string -// ---------------------------------------------------------------------------- - -inline std::string msgpack_schema_to_string(auto const &obj) { - msgpack::sbuffer output; - SchemaPacker printer{output}; - schema_pack(printer, obj); - msgpack::object_handle oh = msgpack::unpack(output.data(), output.size()); - std::stringstream pretty; - pretty << oh.get() << std::endl; - return pretty.str(); -} - -} // namespace ipc diff --git a/ipc-codegen/test/schema_visitor.test.ts b/ipc-codegen/test/schema_visitor.test.ts index d48b73c5ff87..90cfd19e9db6 100644 --- a/ipc-codegen/test/schema_visitor.test.ts +++ b/ipc-codegen/test/schema_visitor.test.ts @@ -3,7 +3,11 @@ * node --experimental-strip-types --no-warnings test/schema_visitor.test.ts * Exits non-zero on failure. */ -import { SchemaVisitor } from "../src/schema_visitor.ts"; +import { + SchemaVisitor, + stripJsonc, + friendlyToPositional, +} from "../src/schema_visitor.ts"; import * as fs from "node:fs"; import * as path from "node:path"; @@ -41,10 +45,11 @@ const errResp = ["FooErrorResponse", { message: "string" }]; expectOk("echo schema is valid", () => { const schemaPath = path.join( import.meta.dirname, - "../echo_example/schema/schema.json", + "../echo_example/schema/schema.jsonc", ); - const schema = JSON.parse(fs.readFileSync(schemaPath, "utf8")); - new SchemaVisitor().visit(schema.commands, schema.responses); + const parsed = JSON.parse(stripJsonc(fs.readFileSync(schemaPath, "utf8"))); + const { commands, responses } = friendlyToPositional(parsed); + new SchemaVisitor().visit(commands, responses); }); expectThrows(