Skip to content

Add pb2json depth limit #3099

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wwbmmm merged 4 commits into from
Sep 24, 2025
Merged

Add pb2json depth limit #3099

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
879f041
Add pb2json depth limit
wwbmmm Sep 20, 2025
cba4d3a
fix multi json test case failed
wwbmmm Sep 20, 2025
35e3e75
define DEEP_RECURSION_TEST_DEPTH constant
wwbmmm Sep 22, 2025
02410a0
Remove duplicate root_msg and root_val param
wwbmmm Sep 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/butil/third_party/rapidjson/reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1536,9 +1536,15 @@ class GenericReader {
state = d;

// Do not further consume streams if a root JSON has been parsed.
if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState)
if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState) {
// wwb: Update parseResult_.Offset() when kParseStopWhenDoneFlag
// is set which means the user needs to know where to resume
// parsing in next calls to JsonToProtoMessage()
if (is.Peek() != '\0') {
SetParseError(kParseErrorNone, is.Tell());
}
break;

}
SkipWhitespace(is);
}

Expand Down
48 changes: 30 additions & 18 deletions src/json2pb/json_to_pb.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include <google/protobuf/descriptor.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <gflags/gflags.h>
#include "butil/strings/string_number_conversions.h"
#include "butil/third_party/rapidjson/error/error.h"
#include "butil/third_party/rapidjson/rapidjson.h"
Expand Down Expand Up @@ -59,6 +60,12 @@

namespace json2pb {

// Use iterative parsing to avoid stack overflow.
const int RAPIDJSON_PARSE_FLAG_DEFAULT = BUTIL_RAPIDJSON_NAMESPACE::kParseIterativeFlag;
const int RAPIDJSON_PARSE_FLAG_STOP_WHEN_DONE = BUTIL_RAPIDJSON_NAMESPACE::kParseStopWhenDoneFlag | RAPIDJSON_PARSE_FLAG_DEFAULT;

DEFINE_int32(json2pb_max_recursion_depth, 100, "Maximum recursion depth of JSON parser");

Json2PbOptions::Json2PbOptions()
#ifdef BAIDU_INTERNAL
: base64_to_bytes(false)
Expand Down Expand Up @@ -284,8 +291,7 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
google::protobuf::Message* message,
const Json2PbOptions& options,
std::string* err,
bool root_val = false);

int depth);
//Json value to protobuf convert rules for type:
//Json value type Protobuf type convert rules
//int int uint int64 uint64 valid convert is available
Expand Down Expand Up @@ -314,7 +320,8 @@ static bool JsonValueToProtoField(const BUTIL_RAPIDJSON_NAMESPACE::Value& value,
const google::protobuf::FieldDescriptor* field,
google::protobuf::Message* message,
const Json2PbOptions& options,
std::string* err) {
std::string* err,
int depth) {
if (value.IsNull()) {
if (field->is_required()) {
J2PERROR(err, "Missing required field: %s", field->full_name().c_str());
Expand Down Expand Up @@ -477,13 +484,13 @@ static bool JsonValueToProtoField(const BUTIL_RAPIDJSON_NAMESPACE::Value& value,
const BUTIL_RAPIDJSON_NAMESPACE::Value& item = value[index];
if (TYPE_MATCH == J2PCHECKTYPE(item, message, Object)) {
if (!JsonValueToProtoMessage(
item, reflection->AddMessage(message, field), options, err)) {
item, reflection->AddMessage(message, field), options, err, depth + 1)) {
return false;
}
}
}
} else if (!JsonValueToProtoMessage(
value, reflection->MutableMessage(message, field), options, err)) {
value, reflection->MutableMessage(message, field), options, err, depth + 1)) {
return false;
}
break;
Expand All @@ -495,7 +502,8 @@ bool JsonMapToProtoMap(const BUTIL_RAPIDJSON_NAMESPACE::Value& value,
const google::protobuf::FieldDescriptor* map_desc,
google::protobuf::Message* message,
const Json2PbOptions& options,
std::string* err) {
std::string* err,
int depth) {
if (!value.IsObject()) {
J2PERROR(err, "Non-object value for map field: %s",
map_desc->full_name().c_str());
Expand All @@ -515,7 +523,7 @@ bool JsonMapToProtoMap(const BUTIL_RAPIDJSON_NAMESPACE::Value& value,
entry_reflection->SetString(
entry, key_desc, std::string(it->name.GetString(),
it->name.GetStringLength()));
if (!JsonValueToProtoField(it->value, value_desc, entry, options, err)) {
if (!JsonValueToProtoField(it->value, value_desc, entry, options, err, depth + 1)) {
return false;
}
}
Expand All @@ -526,10 +534,14 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
google::protobuf::Message* message,
const Json2PbOptions& options,
std::string* err,
bool root_val) {
int depth) {
if (depth > FLAGS_json2pb_max_recursion_depth) {
J2PERROR_WITH_PB(message, err, "Exceeded maximum recursion depth");
return false;
}
const google::protobuf::Descriptor* descriptor = message->GetDescriptor();
if (!json_value.IsObject() &&
!(json_value.IsArray() && options.array_to_single_repeated && root_val)) {
!(json_value.IsArray() && options.array_to_single_repeated && depth == 0)) {
J2PERROR_WITH_PB(message, err, "The input is not a json object");
return false;
}
Expand Down Expand Up @@ -560,7 +572,7 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,

if (json_value.IsArray()) {
if (fields.size() == 1 && fields.front()->is_repeated()) {
return JsonValueToProtoField(json_value, fields.front(), message, options, err);
return JsonValueToProtoField(json_value, fields.front(), message, options, err, depth);
}

J2PERROR_WITH_PB(message, err, "the input json can't be array here");
Expand Down Expand Up @@ -602,11 +614,11 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,

if (IsProtobufMap(field) && value_ptr->IsObject()) {
// Try to parse json like {"key":value, ...} into protobuf map
if (!JsonMapToProtoMap(*value_ptr, field, message, options, err)) {
if (!JsonMapToProtoMap(*value_ptr, field, message, options, err, depth)) {
return false;
}
} else {
if (!JsonValueToProtoField(*value_ptr, field, message, options, err)) {
if (!JsonValueToProtoField(*value_ptr, field, message, options, err, depth)) {
return false;
}
}
Expand All @@ -624,12 +636,12 @@ inline bool JsonToProtoMessageInline(const std::string& json_string,
}
BUTIL_RAPIDJSON_NAMESPACE::Document d;
if (options.allow_remaining_bytes_after_parsing) {
d.Parse<BUTIL_RAPIDJSON_NAMESPACE::kParseStopWhenDoneFlag>(json_string.c_str());
d.Parse<RAPIDJSON_PARSE_FLAG_STOP_WHEN_DONE>(json_string.c_str());
if (parsed_offset != nullptr) {
*parsed_offset = d.GetErrorOffset();
}
} else {
d.Parse<0>(json_string.c_str());
d.Parse<RAPIDJSON_PARSE_FLAG_DEFAULT>(json_string.c_str());
}
if (d.HasParseError()) {
if (options.allow_remaining_bytes_after_parsing) {
Expand All @@ -642,7 +654,7 @@ inline bool JsonToProtoMessageInline(const std::string& json_string,
J2PERROR_WITH_PB(message, error, "Invalid json: %s", BUTIL_RAPIDJSON_NAMESPACE::GetParseError_En(d.GetParseError()));
return false;
}
return JsonValueToProtoMessage(d, message, options, error, true);
return JsonValueToProtoMessage(d, message, options, error, 0);
}

bool JsonToProtoMessage(const std::string& json_string,
Expand Down Expand Up @@ -672,12 +684,12 @@ bool JsonToProtoMessage(ZeroCopyStreamReader* reader,
}
BUTIL_RAPIDJSON_NAMESPACE::Document d;
if (options.allow_remaining_bytes_after_parsing) {
d.ParseStream<BUTIL_RAPIDJSON_NAMESPACE::kParseStopWhenDoneFlag, BUTIL_RAPIDJSON_NAMESPACE::UTF8<>>(*reader);
d.ParseStream<RAPIDJSON_PARSE_FLAG_STOP_WHEN_DONE, BUTIL_RAPIDJSON_NAMESPACE::UTF8<>>(*reader);
if (parsed_offset != nullptr) {
*parsed_offset = d.GetErrorOffset();
}
} else {
d.ParseStream<0, BUTIL_RAPIDJSON_NAMESPACE::UTF8<>>(*reader);
d.ParseStream<RAPIDJSON_PARSE_FLAG_DEFAULT, BUTIL_RAPIDJSON_NAMESPACE::UTF8<>>(*reader);
}
if (d.HasParseError()) {
if (options.allow_remaining_bytes_after_parsing) {
Expand All @@ -690,7 +702,7 @@ bool JsonToProtoMessage(ZeroCopyStreamReader* reader,
J2PERROR_WITH_PB(message, error, "Invalid json: %s", BUTIL_RAPIDJSON_NAMESPACE::GetParseError_En(d.GetParseError()));
return false;
}
return JsonValueToProtoMessage(d, message, options, error, true);
return JsonValueToProtoMessage(d, message, options, error, 0);
}

bool JsonToProtoMessage(const std::string& json_string,
Expand Down
86 changes: 74 additions & 12 deletions src/json2pb/pb_to_json.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <time.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <gflags/gflags.h>
#include "json2pb/zero_copy_stream_writer.h"
#include "json2pb/encode_decode.h"
#include "json2pb/protobuf_map.h"
Expand All @@ -33,6 +34,57 @@
#include "butil/base64.h"

namespace json2pb {

DECLARE_int32(json2pb_max_recursion_depth);

// Helper function to check if the maximum depth of a message is exceeded.
bool ExceedMaxDepth(const google::protobuf::Message& message, int current_depth) {
Comment thread
wwbmmm marked this conversation as resolved.
if (current_depth > FLAGS_json2pb_max_recursion_depth) {
return true;
}

const google::protobuf::Descriptor* descriptor = message.GetDescriptor();
const google::protobuf::Reflection* reflection = message.GetReflection();

std::vector<const google::protobuf::FieldDescriptor*> fields;
// Collect declared fields.
for (int i = 0; i < descriptor->field_count(); ++i) {
fields.push_back(descriptor->field(i));
}
// Collect extension fields (if any).
{
std::vector<const google::protobuf::FieldDescriptor*> ext_fields;
descriptor->file()->pool()->FindAllExtensions(descriptor, &ext_fields);
fields.insert(fields.end(), ext_fields.begin(), ext_fields.end());
}

for (const auto* field : fields) {
if (field->cpp_type() != google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
continue;
}

if (field->is_repeated()) {
const int count = reflection->FieldSize(message, field);
for (int j = 0; j < count; ++j) {
const google::protobuf::Message& sub_message =
reflection->GetRepeatedMessage(message, field, j);
if (ExceedMaxDepth(sub_message, current_depth + 1)) {
return true;
}
}
} else {
if (reflection->HasField(message, field)) {
const google::protobuf::Message& sub_message =
reflection->GetMessage(message, field);
if (ExceedMaxDepth(sub_message, current_depth + 1)) {
return true;
}
}
}
}
return false;
}

Pb2JsonOptions::Pb2JsonOptions()
: enum_option(OUTPUT_ENUM_BY_NAME)
, pretty_json(false)
Expand All @@ -52,22 +104,26 @@ class PbToJsonConverter {
explicit PbToJsonConverter(const Pb2JsonOptions& opt) : _option(opt) {}

template <typename Handler>
bool Convert(const google::protobuf::Message& message, Handler& handler, bool root_msg = false);
bool Convert(const google::protobuf::Message& message, Handler& handler, int depth = 0);

const std::string& ErrorText() const { return _error; }

private:
template <typename Handler>
bool _PbFieldToJson(const google::protobuf::Message& message,
const google::protobuf::FieldDescriptor* field,
Handler& handler);
Handler& handler, int depth);

std::string _error;
Pb2JsonOptions _option;
};

template <typename Handler>
bool PbToJsonConverter::Convert(const google::protobuf::Message& message, Handler& handler, bool root_msg) {
bool PbToJsonConverter::Convert(const google::protobuf::Message& message, Handler& handler, int depth) {
if (depth > FLAGS_json2pb_max_recursion_depth) {
_error = "Exceeded maximum recursion depth";
return false;
}
const google::protobuf::Reflection* reflection = message.GetReflection();
const google::protobuf::Descriptor* descriptor = message.GetDescriptor();

Expand Down Expand Up @@ -101,9 +157,9 @@ bool PbToJsonConverter::Convert(const google::protobuf::Message& message, Handle
}
}

if (root_msg && _option.single_repeated_to_array) {
if (depth == 0 && _option.single_repeated_to_array) {
if (map_fields.empty() && fields.size() == 1 && fields.front()->is_repeated()) {
return _PbFieldToJson(message, fields.front(), handler);
return _PbFieldToJson(message, fields.front(), handler, depth);
}
}

Expand Down Expand Up @@ -134,7 +190,7 @@ bool PbToJsonConverter::Convert(const google::protobuf::Message& message, Handle
bool decoded = decode_name(orig_name, field_name_str);
const std::string& name = decoded ? field_name_str : orig_name;
handler.Key(name.data(), name.size(), false);
if (!_PbFieldToJson(message, field, handler)) {
if (!_PbFieldToJson(message, field, handler, depth)) {
return false;
}
}
Expand Down Expand Up @@ -164,7 +220,7 @@ bool PbToJsonConverter::Convert(const google::protobuf::Message& message, Handle
handler.Key(entry_name.data(), entry_name.size(), false);

// Fill in entries into this json object
if (!_PbFieldToJson(entry, value_desc, handler)) {
if (!_PbFieldToJson(entry, value_desc, handler, depth)) {
return false;
}
}
Expand All @@ -180,7 +236,7 @@ template <typename Handler>
bool PbToJsonConverter::_PbFieldToJson(
const google::protobuf::Message& message,
const google::protobuf::FieldDescriptor* field,
Handler& handler) {
Handler& handler, int depth) {
const google::protobuf::Reflection* reflection = message.GetReflection();
switch (field->cpp_type()) {
#define CASE_FIELD_TYPE(cpptype, method, valuetype, handle) \
Expand Down Expand Up @@ -280,14 +336,14 @@ bool PbToJsonConverter::_PbFieldToJson(
handler.StartArray();
for (int index = 0; index < field_size; ++index) {
if (!Convert(reflection->GetRepeatedMessage(
message, field, index), handler)) {
message, field, index), handler, depth + 1)) {
return false;
}
}
handler.EndArray(field_size);

} else {
if (!Convert(reflection->GetMessage(message, field), handler)) {
if (!Convert(reflection->GetMessage(message, field), handler, depth + 1)) {
return false;
}
}
Expand All @@ -305,10 +361,10 @@ bool ProtoMessageToJsonStream(const google::protobuf::Message& message,
bool succ = false;
if (options.pretty_json) {
BUTIL_RAPIDJSON_NAMESPACE::PrettyWriter<OutputStream> writer(os);
succ = converter.Convert(message, writer, true);
succ = converter.Convert(message, writer);
} else {
BUTIL_RAPIDJSON_NAMESPACE::OptimizedWriter<OutputStream> writer(os);
succ = converter.Convert(message, writer, true);
succ = converter.Convert(message, writer);
}
if (!succ && error) {
error->clear();
Expand Down Expand Up @@ -352,6 +408,12 @@ bool ProtoMessageToJson(const google::protobuf::Message& message,
bool ProtoMessageToProtoJson(const google::protobuf::Message& message,
google::protobuf::io::ZeroCopyOutputStream* json,
const Pb2ProtoJsonOptions& options, std::string* error) {
if (ExceedMaxDepth(message, 0)) {
if (error) {
*error = "Exceeded maximum recursion depth";
}
return false;
}
butil::IOBuf buf;
butil::IOBufAsZeroCopyOutputStream output_stream(&buf);
if (!message.SerializeToZeroCopyStream(&output_stream)) {
Expand Down
Loading
Loading