Use stream json api in protobuf 31 and later versions

Author: chenBright

.github/workflows/ci-linux.yml

@@ -15,192 +15,14 @@ env:
 
 # https://github.com/actions/runner-images
 jobs:
-  compile-with-make:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-all-dependencies
-
-    - name: gcc with default options
-      uses: ./.github/actions/compile-with-make
-      with:
-        options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=gcc --cxx=g++ --werror
-
-    - name: gcc with all options
-      uses: ./.github/actions/compile-with-make
-      with:
-        options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=gcc --cxx=g++ --werror --with-thrift --with-glog --with-rdma  --with-debug-bthread-sche-safety --with-debug-lock --with-bthread-tracer --with-asan
-
-    - name: clang with default options
-      uses: ./.github/actions/compile-with-make
-      with:
-        options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=clang --cxx=clang++ --werror
-
-    - name: clang with all options
-      uses: ./.github/actions/compile-with-make
-      with:
-        options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=clang --cxx=clang++ --werror --with-thrift --with-glog --with-rdma  --with-debug-bthread-sche-safety --with-debug-lock --with-bthread-tracer --with-asan
-
-  compile-with-cmake:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-all-dependencies
-
-    - name: gcc with default options
-      run: |
-        export CC=gcc && export CXX=g++
-        mkdir gcc_build && cd gcc_build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ..
-        make -j ${{env.proc_num}} && make clean
-
-    - name: gcc with all options
-      run: |
-        export CC=gcc && export CXX=g++
-        mkdir gcc_build_all && cd gcc_build_all
-        cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON -DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON -DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ..
-        make -j ${{env.proc_num}} && make clean
-
-    - name: clang with default options
-      run: |
-        export CC=clang && export CXX=clang++
-        mkdir clang_build && cd clang_build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ..
-        make -j ${{env.proc_num}} && make clean
-
-    - name: clang with all options
-      run: |
-        export CC=clang && export CXX=clang++
-        mkdir clang_build_all && cd clang_build_all
-        cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON -DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON -DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ..
-        make -j ${{env.proc_num}} && make clean
-
-  gcc-compile-with-make-protobuf:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-essential-dependencies
-
-    - name: protobuf 3.5.1
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 3.5.1
-        protobuf-cpp-version: 3.5.1
-        protobuf-install-dir: /protobuf-3.5.1
-        config-brpc-options: --cc=gcc --cxx=g++ --werror
-
-    - name: protobuf 3.12.4
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 3.12.4
-        protobuf-cpp-version: 3.12.4
-        protobuf-install-dir: /protobuf-3.12.4
-        config-brpc-options: --cc=gcc --cxx=g++ --werror
-
-    - name: protobuf 21.12
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 21.12
-        protobuf-cpp-version: 3.21.12
-        protobuf-install-dir: /protobuf-3.21.12
-        config-brpc-options: --cc=gcc --cxx=g++ --werror
-
   gcc-compile-with-bazel:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
     - run: bazel build --verbose_failures -- //... -//example/...
 
-  gcc-compile-with-boringssl:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - run: bazel build --verbose_failures --define with_mesalink=false --define with_glog=true --define with_thrift=true --define BRPC_WITH_BORINGSSL=true -- //... -//example/...
-
-  gcc-compile-with-bazel-all-options:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - run: bazel build --verbose_failures --define with_mesalink=false --define with_glog=true --define with_thrift=true --define with_debug_bthread_sche_safety=true --define with_debug_lock=true --define with_asan=true -- //... -//example/...
-
-  clang-compile-with-make-protobuf:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-essential-dependencies
-
-    - name: protobuf 3.5.1
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 3.5.1
-        protobuf-cpp-version: 3.5.1
-        protobuf-install-dir: /protobuf-3.5.1
-        config-brpc-options: --cc=clang --cxx=clang++ --werror
-
-    - name: protobuf 3.12.4
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 3.12.4
-        protobuf-cpp-version: 3.12.4
-        protobuf-install-dir: /protobuf-3.12.4
-        config-brpc-options: --cc=clang --cxx=clang++ --werror
-
-    - name: protobuf 21.12
-      uses: ./.github/actions/compile-with-make-protobuf
-      with:
-        protobuf-version: 21.12
-        protobuf-cpp-version: 3.21.12
-        protobuf-install-dir: /protobuf-3.21.12
-        config-brpc-options: --cc=clang --cxx=clang++ --werror
-
   clang-compile-with-bazel:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v2
     - run: bazel build --verbose_failures --action_env=CC=clang -- //... -//example/...
-
-  clang-compile-with-boringssl:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - run: bazel build --verbose_failures --action_env=CC=clang --define with_mesalink=false --define with_glog=true --define with_thrift=true --define BRPC_WITH_BORINGSSL=true -- //... -//example/...
-
-  clang-compile-with-bazel-all-options:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - run: bazel build --verbose_failures --action_env=CC=clang --define with_mesalink=false --define with_glog=true --define with_thrift=true --define with_debug_bthread_sche_safety=true --define with_debug_lock=true --define with_asan=true -- //... -//example/...
-
-  clang-unittest:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-essential-dependencies
-    - uses: ./.github/actions/init-ut-make-config
-      with:
-        options: --cc=clang-12 --cxx=clang++-12 --with-bthread-tracer
-    - name: compile tests
-      run: |
-        cat config.mk
-        cd test
-        make -j ${{env.proc_num}}
-    - name: run tests
-      run: |
-        cd test
-        sh ./run_tests.sh
-
-  clang-unittest-asan:
-    runs-on: ubuntu-22.04
-    steps:
-    - uses: actions/checkout@v2
-    - uses: ./.github/actions/install-essential-dependencies
-    - uses: ./.github/actions/init-ut-make-config
-      with:
-        options: --cc=clang-12 --cxx=clang++-12 --with-bthread-tracer --with-asan
-    - name: compile tests
-      run: |
-        cat config.mk
-        cd test
-        make NEED_GPERFTOOLS=0 -j ${{env.proc_num}}
-    - name: run tests
-      run: |
-        cd test
-        sh ./run_tests.sh

.github/workflows/ci-macos.yml

@@ -8,7 +8,7 @@ module(
 bazel_dep(name = 'abseil-cpp', version = '20210324.2', repo_name = 'com_google_absl')
 bazel_dep(name = 'bazel_skylib', version = '1.0.3')
 bazel_dep(name = 'boringssl', version = '0.0.0-20211025-d4f1ab9')
-bazel_dep(name = 'protobuf', version = '27.3', repo_name = 'com_google_protobuf')
+bazel_dep(name = 'protobuf', version = '31.0', repo_name = 'com_google_protobuf')
 bazel_dep(name = 'gflags', version = '2.2.2', repo_name = 'com_github_gflags_gflags')
 bazel_dep(name = 'glog', version = '0.5.0', repo_name = 'com_github_google_glog')
 bazel_dep(name = 'platforms', version = '0.0.4')

MODULE.bazel

@@ -719,6 +719,14 @@ bool ProtoJsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream* json,
                              google::protobuf::Message* message,
                              const ProtoJson2PbOptions& options,
                              std::string* error) {
+#if GOOGLE_PROTOBUF_VERSION >= 6031000
+    auto st = google::protobuf::json::JsonStreamToMessage(json, message, options);
+    bool ok = st.ok();
+    if (!ok && NULL != error) {
+        *error = st.ToString();
+    }
+    return st.ok();
+#else
     TypeResolverUniqueptr type_resolver = GetTypeResolver(*message);
     std::string type_url = GetTypeUrl(*message);
     butil::IOBuf buf;
@@ -739,6 +747,7 @@ bool ProtoJsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream* json,
         *error = "Fail to ParseFromCodedStream";
     }
     return ok;
+#endif // GOOGLE_PROTOBUF_VERSION >= 6031000
 }
 
 bool ProtoJsonToProtoMessage(const std::string& json, google::protobuf::Message* message,

src/json2pb/json_to_pb.cpp

@@ -352,6 +352,14 @@ bool ProtoMessageToJson(const google::protobuf::Message& message,
 bool ProtoMessageToProtoJson(const google::protobuf::Message& message,
                              google::protobuf::io::ZeroCopyOutputStream* json,
                              const Pb2ProtoJsonOptions& options, std::string* error) {
+#if GOOGLE_PROTOBUF_VERSION >= 6031000
+    auto st = google::protobuf::util::MessageToJson(message, json, options);
+    bool ok = st.ok();
+    if (!ok && NULL != error) {
+        *error = st.ToString();
+    }
+    return st.ok();
+#else
     butil::IOBuf buf;
     butil::IOBufAsZeroCopyOutputStream output_stream(&buf);
     if (!message.SerializeToZeroCopyStream(&output_stream)) {
@@ -368,6 +376,7 @@ bool ProtoMessageToProtoJson(const google::protobuf::Message& message,
         *error = st.ToString();
     }
     return ok;
+#endif // GOOGLE_PROTOBUF_VERSION >= 6031000
 }
 
 bool ProtoMessageToProtoJson(const google::protobuf::Message& message, std::string* json,