Merge remote-tracking branch 'origin/30-pb-producer' into staging

Author: steven-varga

src/consumer_pb.hpp

@@ -21,6 +21,8 @@
 #include "consumer_rlp.hpp"
 #include "consumer_bson.hpp"
 #include "consumer_avro.hpp"
+#include "producer_pb.hpp"
+#include "consumer_pb.hpp"
 #include "h5_attr_translator.hpp"
 #include "consumer_json.hpp"
 #include "consumer_msgpack.hpp"
@@ -52,6 +54,24 @@ clang::ast_matchers::StatementMatcher h5templateMatcher = clang::ast_matchers::c
 
 enum class OutputFormat { hdf5, protobuf, json, msgpack, cbor, bson, avro, rlp,
                           sql_postgres, sql_mysql, sql_lite3 };
+// pbTemplateMatcher: same shape as h5templateMatcher, but triggers on the
+// pb.hpp public surface. Covers pb::encode, pb::decode, and pb::encode_into.
+clang::ast_matchers::StatementMatcher pbTemplateMatcher = clang::ast_matchers::callExpr( clang::ast_matchers::allOf(
+	clang::ast_matchers::hasDescendant( clang::ast_matchers::declRefExpr( clang::ast_matchers::to( clang::ast_matchers::varDecl().bind("variableDecl")  ) ) ),
+	clang::ast_matchers::hasDescendant( clang::ast_matchers::declRefExpr( clang::ast_matchers::to(
+		clang::ast_matchers::functionDecl( clang::ast_matchers::allOf(
+			clang::ast_matchers::eachOf(
+				clang::ast_matchers::hasName("pb::encode"),  clang::ast_matchers::hasName("pb::decode"),
+				clang::ast_matchers::hasName("pb::encode_into")
+			),
+			clang::ast_matchers::hasTemplateArgument(0,  clang::ast_matchers::refersToType( clang::ast_matchers::qualType(
+				clang::ast_matchers::hasDeclaration( clang::ast_matchers::cxxRecordDecl(clang::ast_matchers::isStruct()).bind("cxxRecordDecl"))
+			) )),
+			clang::ast_matchers::isTemplateInstantiation()
+	))  )))
+));
+
+enum class OutputFormat { hdf5, protobuf, json, msgpack, cbor, bson, avro, rlp };
 
 static llvm::cl::OptionCategory MyToolCategory("h5cpp options");
 static llvm::cl::extrahelp CommonHelp(clang::tooling::CommonOptionsParser::HelpMessage);
@@ -123,14 +143,21 @@ int main(int argc, const char **argv) {
 				rc = Tool.run(clang::tooling::newFrontendActionFactory(&Finder).get());
 				break;
 			}
-			case OutputFormat::protobuf:
-				llvm::errs() << "h5cpp-compiler: --format protobuf not yet implemented\n";
-				rc = 1;
+			case OutputFormat::protobuf: {
+				PbTemplateCallback<PbProducer> callback(work_path);
+				clang::ast_matchers::MatchFinder Finder;
+				Finder.addMatcher(pbTemplateMatcher, &callback);
+				rc = Tool.run(clang::tooling::newFrontendActionFactory(&Finder).get());
+				if (rc == 0 && callback.error()) rc = 1;
 				break;
 			case OutputFormat::json: {
 				JsonTemplateCallback callback(work_path);
 				Finder.addMatcher(h5templateMatcher, &callback);
 				rc = Tool.run(clang::tooling::newFrontendActionFactory(&Finder).get());
+			}
+			case OutputFormat::json:
+				llvm::errs() << "h5cpp-compiler: --format json not yet implemented\n";
+				rc = 1;
 				break;
 			}
 			case OutputFormat::msgpack:

src/h5cpp.cpp

@@ -0,0 +1,142 @@
+/* Copyright (c) 2018-2026 Steven Varga, steven@vargalabs.com Toronto, ON Canada */
+
+#pragma once
+
+#include "producer.hpp"
+
+#include <cstdint>
+#include <map>
+#include <ostream>
+#include <string>
+#include <utility>
+#include <vector>
+
+// Tier-1 backend for vargalabs/sandbox/pb.hpp. Emits:
+//   template<> struct pb::meta::descriptor_t<T> {
+//       static constexpr auto fields = std::tuple{
+//           pb::field<N, &T::m>{}, ...
+//       };
+//   };
+// One specialization per encountered record type, in topological order
+// (nested types before outer types). Field numbers come from per-field
+// [[clang::annotate("pb::field=N")]] attributes — the consumer (consumer_pb.hpp)
+// parses the annotations and supplies N to type_insert.
+struct PbProducer : Producer<PbProducer> {
+
+    void file_begin_impl(){
+        io << "/* Generated by h5cpp-compiler --protocol-buffers. Do not edit. */\n";
+        io << "#pragma once\n\n";
+        io << "#include <pb.hpp>\n\n";
+    }
+
+    void file_end_impl(){
+        io << "\n";
+    }
+
+    // template_decl is the per-record opener. `record` is the qualified type
+    // name (e.g. "::sn::sensor::reading_t"). Each record's descriptor_t is a
+    // standalone specialization at file scope.
+    void template_decl_impl(const std::string& record){
+        record_name = record;
+        first_field = true;
+        io << "template<> struct pb::meta::descriptor_t<" << record << "> {\n";
+        io << "    static constexpr auto fields = std::tuple{";
+    }
+
+    // record_decl is called for each topologically-prior record (nested
+    // types). For PbProducer we don't emit nested compound declarations —
+    // nested message types get their OWN descriptor_t specialization in a
+    // prior pass. The variable name is unused in pb output but tracked for
+    // the cache contract.
+    void record_decl_impl(const std::string& var, const std::string& /*record_name*/){
+        (void)var;
+        // no-op for pb; nested messages reference their own descriptor_t<T>
+    }
+
+    // array_decl: C arrays (T[N]) don't have a clean proto3 wire mapping.
+    // Stage 1 refuses; users must use std::vector<T> (FR3 repeated) or
+    // std::array<T,N> (planned future).
+    void array_decl_impl(const std::string& var, const std::string& /*type*/, uint64_t /*size*/){
+        (void)var;
+        io << "\n    // ERROR: C arrays not supported by pb backend (use std::vector<T> instead)";
+    }
+
+    // type_insert: emit one pb::field<N, &Record::member>{} entry.
+    // `var` is overloaded by the consumer to carry the field number as a
+    // decimal string ("1", "2", ...) instead of the HDF5 vname. record_name
+    // is the qualified record type; field_name is the member identifier.
+    void type_insert_impl(const std::string& var, const std::string& field_name,
+                          const std::string& rec_name, const std::string& /*type*/){
+        if (!first_field) io << ",";
+        io << "\n        pb::field<" << var << ", &" << rec_name << "::" << field_name << ">{}";
+        first_field = false;
+    }
+
+    void return_type_impl(const std::string& /*var*/){
+        io << "\n    };\n};\n\n";
+    }
+
+    void type_release_impl(){
+        // pb has no resource-id lifetime to manage — descriptor_t entries are
+        // pure compile-time. No-op.
+    }
+
+    // The cache contract is HDF5-shaped (cpp-type → hid_t-var mapping).
+    // PbProducer doesn't need that vocabulary — pb dispatches via traits at
+    // the library level, not via per-type variable aliases. Implement as
+    // identity stubs so the existing consumer driver still compiles.
+    bool cache_add_impl(const std::string& /*key*/, const std::string& /*type*/){ return true; }
+    std::string cache_impl(const std::string& type){ return type; }
+
+    // FR6/stage 4: oneof emission. Called directly by PbTemplateCallback for
+    // std::variant member fields (not via the CRTP Producer<> hook surface,
+    // which is HDF5-shaped). Emits a pb::oneof<&R::m, pb::alt<T1,N1>,...>{}
+    // entry in the descriptor's fields tuple.
+    void emit_oneof(const std::string& member_name,
+                    const std::string& record_name,
+                    const std::vector<std::string>& alt_types,
+                    const std::vector<std::uint32_t>& tags){
+        if (!first_field) io << ",";
+        io << "\n        pb::oneof<&" << record_name << "::" << member_name;
+        for (std::size_t i = 0; i < alt_types.size(); ++i) {
+            io << ", pb::alt<" << alt_types[i] << ", " << tags[i] << ">";
+        }
+        io << ">{}";
+        first_field = false;
+    }
+
+    // Stage 5: wire-spec annotation. Same shape as type_insert but emits an
+    // explicit WireSpec template arg on pb::field<N, Ptr, WireSpec>. Called
+    // when the consumer sees a [[clang::annotate("pb::wire=spec")]] attribute
+    // on a field. wire_spec is the spec name without the `_t` suffix, e.g.
+    // "sint32", "fixed64", "sfixed32".
+    void emit_field_with_spec(std::uint32_t tag,
+                              const std::string& field_name,
+                              const std::string& record_name,
+                              const std::string& wire_spec){
+        if (!first_field) io << ",";
+        io << "\n        pb::field<" << tag
+           << ", &" << record_name << "::" << field_name
+           << ", pb::wire::" << wire_spec << "_t>{}";
+        first_field = false;
+    }
+
+    // Stage 6 (Tier 4): adapter_field emission. Called when the consumer
+    // sees a [[clang::annotate("pb::adapter=Name")]] attribute on a field.
+    // Name is the adapter shorthand ("Timestamp", "Duration"), which maps
+    // to the library symbol pb::Name_adapter (Timestamp_adapter, etc.).
+    void emit_adapter_field(std::uint32_t tag,
+                             const std::string& field_name,
+                             const std::string& record_name,
+                             const std::string& adapter_name){
+        if (!first_field) io << ",";
+        io << "\n        pb::adapter_field<" << tag
+           << ", &" << record_name << "::" << field_name
+           << ", pb::" << adapter_name << "_adapter>{}";
+        first_field = false;
+    }
+
+private:
+    std::string record_name;
+    bool        first_field = true;
+};

src/producer_pb.hpp

@@ -77,6 +77,19 @@ foreach(fixture IN LISTS FIXTURES)
   add_h5cpp_fixture_test(${fixture})
 endforeach()
 
+set(PB_FIXTURES
+  pb_primitives
+  pb_strings_enums
+  pb_composites
+  pb_variant
+  pb_wire_specs
+  pb_chrono
+)
+
+foreach(fixture IN LISTS PB_FIXTURES)
+  add_h5cpp_backend_fixture_test(${fixture} protobuf)
+endforeach()
+
 set(RLP_FIXTURES
   rlp_primitives
   rlp_arrays

tests/CMakeLists.txt

@@ -0,0 +1,32 @@
+#include "pb_stub.hpp"
+
+#include <chrono>
+#include <cstdint>
+
+// Tier-4 fixture (stage 6): chrono adapter annotation. A user struct with
+// std::chrono::system_clock::time_point + std::chrono::nanoseconds members
+// gets pb::adapter_field<N, Ptr, pb::Timestamp_adapter> / Duration_adapter
+// entries in the emitted descriptor. The library bridges the C++ types
+// to proto3 google.protobuf.Timestamp / Duration wire messages.
+namespace sn::pb_test {
+
+struct event_t {
+    [[clang::annotate("pb::field=1")]]
+        std::int64_t id;
+
+    [[clang::annotate("pb::field=2")]]
+    [[clang::annotate("pb::adapter=Timestamp")]]
+        std::chrono::system_clock::time_point when;
+
+    [[clang::annotate("pb::field=3")]]
+    [[clang::annotate("pb::adapter=Duration")]]
+        std::chrono::nanoseconds ttl;
+};
+
+} // namespace sn::pb_test
+
+void use() {
+    sn::pb_test::event_t e{};
+    auto buf = pb::encode(e);
+    (void)buf;
+}

tests/fixtures/pb_chrono.cpp

@@ -0,0 +1,31 @@
+#include "pb_stub.hpp"
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <vector>
+
+// Tier-3 fixture (stage 3): composite shapes — std::vector (repeated),
+// std::optional (proto3 optional explicit presence). The compiler emits the
+// same pb::field<N, &T::m>{} entries; pb.hpp's trait dispatch handles the
+// shape at compile time via is_pb_repeated_v and is_pb_optional_v.
+//
+// stdlib types are NOT walked into for descriptor emission (only the outer
+// user record gets a descriptor_t<>).
+namespace sn::pb_test {
+
+struct profile_t {
+    [[clang::annotate("pb::field=1")]] std::string                  name;
+    [[clang::annotate("pb::field=2")]] std::vector<std::int32_t>    scores;
+    [[clang::annotate("pb::field=3")]] std::optional<std::string>   nickname;
+    [[clang::annotate("pb::field=4")]] std::vector<std::string>     aliases;
+    [[clang::annotate("pb::field=5")]] std::optional<std::int64_t>  user_id;
+};
+
+} // namespace sn::pb_test
+
+void use() {
+    sn::pb_test::profile_t p{};
+    auto buf = pb::encode(p);
+    (void)buf;
+}

tests/fixtures/pb_composites.cpp

@@ -0,0 +1,25 @@
+#include "pb_stub.hpp"
+
+#include <cstdint>
+
+// Tier-1 fixture: numeric scalars only. Every member carries
+// [[clang::annotate("pb::field=N")]]; the compiler emits a
+// pb::meta::descriptor_t<T> specialization with one pb::field<N,&T::m>{}
+// entry per member, in source-declaration order.
+namespace sn::pb_test {
+struct primitives_t {
+    [[clang::annotate("pb::field=1")]] bool         _bool;
+    [[clang::annotate("pb::field=2")]] std::int32_t _i32;
+    [[clang::annotate("pb::field=3")]] std::int64_t _i64;
+    [[clang::annotate("pb::field=4")]] std::uint32_t _u32;
+    [[clang::annotate("pb::field=5")]] std::uint64_t _u64;
+    [[clang::annotate("pb::field=6")]] float        _f;
+    [[clang::annotate("pb::field=7")]] double       _d;
+};
+} // namespace sn::pb_test
+
+void use() {
+    sn::pb_test::primitives_t p{};
+    auto buf = pb::encode(p);
+    (void)buf;
+}

tests/fixtures/pb_primitives.cpp

@@ -0,0 +1,32 @@
+#include "pb_stub.hpp"
+
+#include <cstdint>
+#include <string>
+
+// Tier-1 fixture (stage 2): non-POD members — std::string is the canonical
+// length-delimited scalar; enum class is the canonical proto3 enum mapping.
+// The compiler emits a single descriptor_t<log_entry_t> with one pb::field
+// entry per member; std::string is NOT walked into (it's a stdlib leaf).
+namespace sn::pb_test {
+
+enum class severity_e : std::int32_t {
+    UNSPECIFIED = 0,
+    INFO        = 1,
+    WARN        = 2,
+    ERROR       = 3,
+};
+
+struct log_entry_t {
+    [[clang::annotate("pb::field=1")]] std::int64_t timestamp_ns;
+    [[clang::annotate("pb::field=2")]] std::string  message;
+    [[clang::annotate("pb::field=3")]] severity_e   level;
+    [[clang::annotate("pb::field=4")]] double       elapsed_ms;
+};
+
+} // namespace sn::pb_test
+
+void use() {
+    sn::pb_test::log_entry_t e{};
+    auto buf = pb::encode(e);
+    (void)buf;
+}

tests/fixtures/pb_strings_enums.cpp

@@ -0,0 +1,26 @@
+#include "pb_stub.hpp"
+
+#include <cstdint>
+#include <string>
+#include <variant>
+
+// Tier-3 fixture (stage 4): std::variant as proto3 oneof. The variant must
+// start with std::monostate (the "absent" state); each subsequent alternative
+// gets a field number from the [[clang::annotate("pb::oneof_tags=N1,N2,…")]]
+// list on the variant member. Compiler emits a pb::oneof<&T::v,
+// pb::alt<T1, N1>, ...>{} entry in the descriptor.
+namespace sn::pb_test {
+
+struct event_t {
+    [[clang::annotate("pb::field=1")]] std::int64_t timestamp_ns;
+    [[clang::annotate("pb::oneof_tags=5,6,7")]]
+        std::variant<std::monostate, std::string, std::int64_t, double> payload;
+};
+
+} // namespace sn::pb_test
+
+void use() {
+    sn::pb_test::event_t e{};
+    auto buf = pb::encode(e);
+    (void)buf;
+}