Use stream json api in protobuf 31 and later versions

chenBright · chenBright · commit 50235ca7f0bc · 2025-05-28T00:10:26.000+08:00
diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml
@@ -15,192 +15,14 @@ env:
 
 # https://github.com/actions/runner-images
 jobs:
-  compile-with-make:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-all-dependencies
-
-    - name: gcc with default options
-      uses: ./.github/actions/compile-with-make
-      with:
-        options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=gcc --cxx=g++ --werror
-
-    - name: gcc with all options
-      uses: ./.github/actions/compile-with-make
-      with:
-        options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=gcc --cxx=g++ --werror --with-thrift --with-glog --with-rdma  --with-debug-bthread-sche-safety --with-debug-lock --with-bthread-tracer --with-asan
-
-    - name: clang with default options
-      uses: ./.github/actions/compile-with-make
-      with:
-        options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=clang --cxx=clang++ --werror
-
-    - name: clang with all options
-      uses: ./.github/actions/compile-with-make
-      with:
-        options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=clang --cxx=clang++ --werror --with-thrift --with-glog --with-rdma  --with-debug-bthread-sche-safety --with-debug-lock --with-bthread-tracer --with-asan
-
-  compile-with-cmake:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-all-dependencies
-
-    - name: gcc with default options
-      run: |
-        export CC=gcc && export CXX=g++
-        mkdir gcc_build && cd gcc_build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ..
-        make -j ${{env.proc_num}} && make clean
-
-    - name: gcc with all options
-      run: |
-        export CC=gcc && export CXX=g++
-        mkdir gcc_build_all && cd gcc_build_all
-        cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON -DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON -DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ..
-        make -j ${{env.proc_num}} && make clean
-
-    - name: clang with default options
-      run: |
-        export CC=clang && export CXX=clang++
-        mkdir clang_build && cd clang_build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ..
-        make -j ${{env.proc_num}} && make clean
-
-    - name: clang with all options
-      run: |
-        export CC=clang && export CXX=clang++
-        mkdir clang_build_all && cd clang_build_all
-        cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON -DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON -DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ..
-        make -j ${{env.proc_num}} && make clean
-
-  gcc-compile-with-make-protobuf:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-essential-dependencies
-
-    - name: protobuf 3.5.1
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 3.5.1
-        protobuf-cpp-version: 3.5.1
-        protobuf-install-dir: /protobuf-3.5.1
-        config-brpc-options: --cc=gcc --cxx=g++ --werror
-
-    - name: protobuf 3.12.4
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 3.12.4
-        protobuf-cpp-version: 3.12.4
-        protobuf-install-dir: /protobuf-3.12.4
-        config-brpc-options: --cc=gcc --cxx=g++ --werror
-
-    - name: protobuf 21.12
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 21.12
-        protobuf-cpp-version: 3.21.12
-        protobuf-install-dir: /protobuf-3.21.12
-        config-brpc-options: --cc=gcc --cxx=g++ --werror
-
   gcc-compile-with-bazel:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
     - run: bazel build --verbose_failures -- //... -//example/...
 
-  gcc-compile-with-boringssl:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - run: bazel build --verbose_failures --define with_mesalink=false --define with_glog=true --define with_thrift=true --define BRPC_WITH_BORINGSSL=true -- //... -//example/...
-
-  gcc-compile-with-bazel-all-options:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - run: bazel build --verbose_failures --define with_mesalink=false --define with_glog=true --define with_thrift=true --define with_debug_bthread_sche_safety=true --define with_debug_lock=true --define with_asan=true -- //... -//example/...
-
-  clang-compile-with-make-protobuf:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-essential-dependencies
-
-    - name: protobuf 3.5.1
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 3.5.1
-        protobuf-cpp-version: 3.5.1
-        protobuf-install-dir: /protobuf-3.5.1
-        config-brpc-options: --cc=clang --cxx=clang++ --werror
-
-    - name: protobuf 3.12.4
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 3.12.4
-        protobuf-cpp-version: 3.12.4
-        protobuf-install-dir: /protobuf-3.12.4
-        config-brpc-options: --cc=clang --cxx=clang++ --werror
-
-    - name: protobuf 21.12
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 21.12
-        protobuf-cpp-version: 3.21.12
-        protobuf-install-dir: /protobuf-3.21.12
-        config-brpc-options: --cc=clang --cxx=clang++ --werror
-
   clang-compile-with-bazel:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
     - run: bazel build --verbose_failures --action_env=CC=clang -- //... -//example/...
-
-  clang-compile-with-boringssl:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - run: bazel build --verbose_failures --action_env=CC=clang --define with_mesalink=false --define with_glog=true --define with_thrift=true --define BRPC_WITH_BORINGSSL=true -- //... -//example/...
-
-  clang-compile-with-bazel-all-options:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - run: bazel build --verbose_failures --action_env=CC=clang --define with_mesalink=false --define with_glog=true --define with_thrift=true --define with_debug_bthread_sche_safety=true --define with_debug_lock=true --define with_asan=true -- //... -//example/...
-
-  clang-unittest:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-essential-dependencies
-    - uses: ./.github/actions/init-ut-make-config
-      with:
-        options: --cc=clang-12 --cxx=clang++-12 --with-bthread-tracer
-    - name: compile tests
-      run: |
-        cat config.mk
-        cd test
-        make -j ${{env.proc_num}}
-    - name: run tests
-      run: |
-        cd test
-        sh ./run_tests.sh
-
-  clang-unittest-asan:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-essential-dependencies
-    - uses: ./.github/actions/init-ut-make-config
-      with:
-        options: --cc=clang-12 --cxx=clang++-12 --with-bthread-tracer --with-asan
-    - name: compile tests
-      run: |
-        cat config.mk
-        cd test
-        make NEED_GPERFTOOLS=0 -j ${{env.proc_num}}
-    - name: run tests
-      run: |
-        cd test
-        sh ./run_tests.sh
diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -8,7 +8,7 @@ module(
 bazel_dep(name = 'abseil-cpp', version = '20210324.2', repo_name = 'com_google_absl')
 bazel_dep(name = 'bazel_skylib', version = '1.0.3')
 bazel_dep(name = 'boringssl', version = '0.0.0-20211025-d4f1ab9')
-bazel_dep(name = 'protobuf', version = '27.3', repo_name = 'com_google_protobuf')
+bazel_dep(name = 'protobuf', version = '31.0', repo_name = 'com_google_protobuf')
 bazel_dep(name = 'gflags', version = '2.2.2', repo_name = 'com_github_gflags_gflags')
 bazel_dep(name = 'glog', version = '0.5.0', repo_name = 'com_github_google_glog')
 bazel_dep(name = 'platforms', version = '0.0.4')
diff --git a/src/butil/iobuf.cpp b/src/butil/iobuf.cpp
@@ -2145,7 +2145,7 @@ void IOBufAsZeroCopyOutputStream::BackUp(int count) {
     LOG_IF(FATAL, count != 0) << "BackUp an empty IOBuf";
 }
 
-google::protobuf::int64 IOBufAsZeroCopyOutputStream::ByteCount() const {
+int64_t IOBufAsZeroCopyOutputStream::ByteCount() const {
     return _byte_count;
 }
 
diff --git a/src/butil/iobuf.h b/src/butil/iobuf.h
@@ -593,7 +593,7 @@ class IOBufAsZeroCopyOutputStream
 
     bool Next(void** data, int* size) override;
     void BackUp(int count) override; // `count' can be as long as ByteCount()
-    google::protobuf::int64 ByteCount() const override;
+    int64_t ByteCount() const override;
 
 private:
     void _release_block();
diff --git a/src/json2pb/json_to_pb.cpp b/src/json2pb/json_to_pb.cpp
@@ -52,8 +52,10 @@
             perr->append(", ", 2);                                      \
         }                                                               \
         butil::string_appendf(perr, fmt, ##__VA_ARGS__);                \
-        if ((pb) != nullptr) {                                            \
-            butil::string_appendf(perr, " [%s]", (pb)->GetDescriptor()->name().c_str());  \
+        if ((pb) != nullptr) {                                          \
+            perr->append("[");                                          \
+            perr->append((pb)->GetDescriptor()->name());                \
+            perr->append("]");                                          \
         }                                                               \
     } else { }
 
@@ -572,7 +574,7 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
     for (size_t i = 0; i < fields.size(); ++i) {
         const google::protobuf::FieldDescriptor* field = fields[i];
         
-        const std::string& orig_name = field->name();
+        const std::string orig_name = field->name();
         bool res = decode_name(orig_name, field_name_str_temp); 
         const std::string& field_name_str = (res ? field_name_str_temp : orig_name);
 
@@ -719,6 +721,14 @@ bool ProtoJsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream* json,
                              google::protobuf::Message* message,
                              const ProtoJson2PbOptions& options,
                              std::string* error) {
+#if GOOGLE_PROTOBUF_VERSION >= 6031000
+    auto st = google::protobuf::json::JsonStreamToMessage(json, message, options);
+    bool ok = st.ok();
+    if (!ok && NULL != error) {
+        *error = st.ToString();
+    }
+    return st.ok();
+#else
     TypeResolverUniqueptr type_resolver = GetTypeResolver(*message);
     std::string type_url = GetTypeUrl(*message);
     butil::IOBuf buf;
@@ -739,6 +749,7 @@ bool ProtoJsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream* json,
         *error = "Fail to ParseFromCodedStream";
     }
     return ok;
+#endif // GOOGLE_PROTOBUF_VERSION >= 6031000
 }
 
 bool ProtoJsonToProtoMessage(const std::string& json, google::protobuf::Message* message,
diff --git a/src/json2pb/pb_to_json.cpp b/src/json2pb/pb_to_json.cpp
@@ -352,6 +352,14 @@ bool ProtoMessageToJson(const google::protobuf::Message& message,
 bool ProtoMessageToProtoJson(const google::protobuf::Message& message,
                              google::protobuf::io::ZeroCopyOutputStream* json,
                              const Pb2ProtoJsonOptions& options, std::string* error) {
+#if GOOGLE_PROTOBUF_VERSION >= 6031000
+    auto st = google::protobuf::util::MessageToJson(message, json, options);
+    bool ok = st.ok();
+    if (!ok && NULL != error) {
+        *error = st.ToString();
+    }
+    return st.ok();
+#else
     butil::IOBuf buf;
     butil::IOBufAsZeroCopyOutputStream output_stream(&buf);
     if (!message.SerializeToZeroCopyStream(&output_stream)) {
@@ -368,6 +376,7 @@ bool ProtoMessageToProtoJson(const google::protobuf::Message& message,
         *error = st.ToString();
     }
     return ok;
+#endif // GOOGLE_PROTOBUF_VERSION >= 6031000
 }
 
 bool ProtoMessageToProtoJson(const google::protobuf::Message& message, std::string* json,
diff --git a/src/json2pb/protobuf_map.cpp b/src/json2pb/protobuf_map.cpp
@@ -38,12 +38,12 @@ bool IsProtobufMap(const FieldDescriptor* field) {
     if (NULL == key_desc
         || key_desc->is_repeated()
         || key_desc->cpp_type() != FieldDescriptor::CPPTYPE_STRING
-        || strcmp(KEY_NAME, key_desc->name().c_str()) != 0) {
+        || key_desc->name() != KEY_NAME) {
         return false;
     }
     const FieldDescriptor* value_desc = entry_desc->field(VALUE_INDEX);
     if (NULL == value_desc
-        || strcmp(VALUE_NAME, value_desc->name().c_str()) != 0) {
+        || value_desc->name() != VALUE_NAME) {
         return false;
     }
     return true;
diff --git a/src/json2pb/protobuf_type_resolver.h b/src/json2pb/protobuf_type_resolver.h
@@ -31,8 +31,9 @@ namespace json2pb {
 #define PROTOBUF_TYPE_URL_PREFIX "type.googleapis.com"
 
 inline std::string GetTypeUrl(const google::protobuf::Message& message) {
-    return butil::string_printf(PROTOBUF_TYPE_URL_PREFIX"/%s",
-                                message.GetDescriptor()->full_name().c_str());
+    std::string type_url = PROTOBUF_TYPE_URL_PREFIX;
+    type_url.append(message.GetDescriptor()->full_name());
+    return type_url;
 }
 
 // unique_ptr deleter for TypeResolver only deletes the object

Original file line number	Diff line number	Diff line change
`@@ -2145,7 +2145,7 @@ void IOBufAsZeroCopyOutputStream::BackUp(int count) {`
`2145`	`2145`	`LOG_IF(FATAL, count != 0) << "BackUp an empty IOBuf";`
`2146`	`2146`	`}`
`2147`	`2147`
`2148`		`-google::protobuf::int64 IOBufAsZeroCopyOutputStream::ByteCount() const {`
	`2148`	`+int64_t IOBufAsZeroCopyOutputStream::ByteCount() const {`
`2149`	`2149`	`return _byte_count;`
`2150`	`2150`	`}`
`2151`	`2151`