Skip to content

Commit fbe99e8

Browse files
mhilsmhils
authored
make protobuf parsing more forgiving (#254)
1 parent ad3fcf5 commit fbe99e8

4 files changed

Lines changed: 125 additions & 70 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
## Unreleased: mitmproxy_rs next
22

3+
- Make gRPC and Protobuf parsing more forgiving.
34

45
## 29 April 2025: mitmproxy_rs 0.12.1
56

‎mitmproxy-contentviews/src/protobuf/view_grpc.rs‎

Lines changed: 103 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
use super::{existing_proto_definitions, reencode};
2+
use crate::protobuf::existing_proto_definitions::DescriptorWithDeps;
23
use crate::{Metadata, Prettify, Protobuf, Reencode};
34
use anyhow::{bail, Context, Result};
45
use flate2::read::{DeflateDecoder, GzDecoder};
6+
use log::info;
57
use mitmproxy_highlight::Language;
68
use serde::Deserialize;
79
use serde_yaml::Value;
@@ -18,11 +20,39 @@ impl Prettify for GRPC {
1820
Language::Yaml
1921
}
2022

21-
fn prettify(&self, mut data: &[u8], metadata: &dyn Metadata) -> Result<String> {
22-
let mut protos = vec![];
23+
fn prettify(&self, data: &[u8], metadata: &dyn Metadata) -> Result<String> {
24+
let encoding = metadata.get_header("grpc-encoding").unwrap_or_default();
25+
let proto_def = existing_proto_definitions::find_best_match(metadata)?;
26+
if let Some(descriptor) = &proto_def {
27+
if let Ok(ret) = self.prettify_with_descriptor(data, &encoding, descriptor) {
28+
return Ok(ret);
29+
}
30+
}
31+
let ret = self.prettify_with_descriptor(data, &encoding, &DescriptorWithDeps::default())?;
32+
if proto_def.is_some() {
33+
info!("Existing gRPC definition does not match, parsing as unknown proto.");
34+
}
35+
Ok(ret)
36+
}
2337

24-
let descriptor = existing_proto_definitions::find_best_match(metadata)?.unwrap_or_default();
38+
fn render_priority(&self, _data: &[u8], metadata: &dyn Metadata) -> f32 {
39+
match metadata.content_type() {
40+
Some("application/grpc") => 1.0,
41+
Some("application/grpc+proto") => 1.0,
42+
Some("application/prpc") => 1.0,
43+
_ => 0.0,
44+
}
45+
}
46+
}
2547

48+
impl GRPC {
49+
fn prettify_with_descriptor(
50+
&self,
51+
mut data: &[u8],
52+
encoding: &str,
53+
descriptor: &DescriptorWithDeps,
54+
) -> Result<String> {
55+
let mut protos = vec![];
2656
while !data.is_empty() {
2757
let compressed = match data[0] {
2858
0 => false,
@@ -39,8 +69,7 @@ impl Prettify for GRPC {
3969

4070
let mut decompressed = Vec::new();
4171
let proto = if compressed {
42-
let encoding = metadata.get_header("grpc-encoding").unwrap_or_default();
43-
match encoding.as_str() {
72+
match encoding {
4473
"deflate" => {
4574
let mut decoder = DeflateDecoder::new(proto);
4675
decoder.read_to_end(&mut decompressed)?;
@@ -57,21 +86,12 @@ impl Prettify for GRPC {
5786
} else {
5887
proto
5988
};
60-
protos.push(Protobuf.prettify_with_descriptor(proto, &descriptor)?);
89+
protos.push(Protobuf.prettify_with_descriptor(proto, descriptor)?);
6190
data = &data[5 + len..];
6291
}
6392

6493
Ok(protos.join("\n---\n\n"))
6594
}
66-
67-
fn render_priority(&self, _data: &[u8], metadata: &dyn Metadata) -> f32 {
68-
match metadata.content_type() {
69-
Some("application/grpc") => 1.0,
70-
Some("application/grpc+proto") => 1.0,
71-
Some("application/prpc") => 1.0,
72-
_ => 0.0,
73-
}
74-
}
7595
}
7696

7797
impl Reencode for GRPC {
@@ -97,7 +117,6 @@ mod tests {
97117
use crate::test::TestMetadata;
98118

99119
const TEST_YAML: &str = "1: 150\n\n---\n\n1: 150\n";
100-
const TEST_YAML_KNOWN: &str = "example: 150\n\n---\n\nexample: 150\n";
101120
const TEST_GRPC: &[u8] = &[
102121
0, 0, 0, 0, 3, 8, 150, 1, // first message
103122
0, 0, 0, 0, 3, 8, 150, 1, // second message
@@ -164,63 +183,81 @@ mod tests {
164183
);
165184
}
166185

167-
#[test]
168-
fn test_existing_proto() {
169-
let metadata = TestMetadata::default().with_protobuf_definitions(concat!(
170-
env!("CARGO_MANIFEST_DIR"),
171-
"/testdata/protobuf/simple.proto"
172-
));
173-
let res = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
174-
assert_eq!(res, TEST_YAML_KNOWN);
175-
}
186+
mod existing_definition {
187+
use super::*;
176188

177-
#[test]
178-
fn test_existing_service_request() {
179-
let metadata = TestMetadata::default()
180-
.with_is_http_request(true)
181-
.with_path("/Service/Method")
182-
.with_protobuf_definitions(concat!(
183-
env!("CARGO_MANIFEST_DIR"),
184-
"/testdata/protobuf/simple_service.proto"
185-
));
186-
let req = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
187-
assert_eq!(req, TEST_YAML);
188-
}
189+
const TEST_YAML_KNOWN: &str = "example: 150\n\n---\n\nexample: 150\n";
189190

190-
#[test]
191-
fn test_existing_service_response() {
192-
let metadata = TestMetadata::default()
193-
.with_is_http_request(false)
194-
.with_path("/Service/Method")
195-
.with_protobuf_definitions(concat!(
191+
#[test]
192+
fn existing_proto() {
193+
let metadata = TestMetadata::default().with_protobuf_definitions(concat!(
196194
env!("CARGO_MANIFEST_DIR"),
197-
"/testdata/protobuf/simple_service.proto"
195+
"/testdata/protobuf/simple.proto"
198196
));
199-
let req = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
200-
assert_eq!(req, TEST_YAML_KNOWN);
201-
}
197+
let res = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
198+
assert_eq!(res, TEST_YAML_KNOWN);
199+
}
202200

203-
#[test]
204-
fn test_existing_package() {
205-
let metadata = TestMetadata::default()
206-
.with_path("/example.simple.Service/Method")
207-
.with_protobuf_definitions(concat!(
208-
env!("CARGO_MANIFEST_DIR"),
209-
"/testdata/protobuf/simple_package.proto"
210-
));
211-
let req = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
212-
assert_eq!(req, TEST_YAML_KNOWN);
213-
}
201+
#[test]
202+
fn existing_service_request() {
203+
let metadata = TestMetadata::default()
204+
.with_is_http_request(true)
205+
.with_path("/Service/Method")
206+
.with_protobuf_definitions(concat!(
207+
env!("CARGO_MANIFEST_DIR"),
208+
"/testdata/protobuf/simple_service.proto"
209+
));
210+
let req = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
211+
assert_eq!(req, TEST_YAML);
212+
}
214213

215-
#[test]
216-
fn test_existing_nested() {
217-
let metadata = TestMetadata::default()
218-
.with_path("/example.nested.Service/Method")
219-
.with_protobuf_definitions(concat!(
214+
#[test]
215+
fn existing_service_response() {
216+
let metadata = TestMetadata::default()
217+
.with_is_http_request(false)
218+
.with_path("/Service/Method")
219+
.with_protobuf_definitions(concat!(
220+
env!("CARGO_MANIFEST_DIR"),
221+
"/testdata/protobuf/simple_service.proto"
222+
));
223+
let req = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
224+
assert_eq!(req, TEST_YAML_KNOWN);
225+
}
226+
227+
#[test]
228+
fn existing_package() {
229+
let metadata = TestMetadata::default()
230+
.with_path("/example.simple.Service/Method")
231+
.with_protobuf_definitions(concat!(
232+
env!("CARGO_MANIFEST_DIR"),
233+
"/testdata/protobuf/simple_package.proto"
234+
));
235+
let req = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
236+
assert_eq!(req, TEST_YAML_KNOWN);
237+
}
238+
239+
#[test]
240+
fn existing_nested() {
241+
let metadata = TestMetadata::default()
242+
.with_path("/example.nested.Service/Method")
243+
.with_protobuf_definitions(concat!(
244+
env!("CARGO_MANIFEST_DIR"),
245+
"/testdata/protobuf/nested.proto"
246+
));
247+
let req = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
248+
assert_eq!(req, TEST_YAML_KNOWN);
249+
}
250+
251+
/// When the existing proto definition does not match the wire data,
252+
/// but the wire data is still valid Protobuf, parse it raw.
253+
#[test]
254+
fn existing_mismatch() {
255+
let metadata = TestMetadata::default().with_protobuf_definitions(concat!(
220256
env!("CARGO_MANIFEST_DIR"),
221-
"/testdata/protobuf/nested.proto"
257+
"/testdata/protobuf/mismatch.proto"
222258
));
223-
let req = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
224-
assert_eq!(req, TEST_YAML_KNOWN);
259+
let res = GRPC.prettify(TEST_GRPC, &metadata).unwrap();
260+
assert_eq!(res, TEST_YAML);
261+
}
225262
}
226263
}

‎mitmproxy-contentviews/src/protobuf/view_protobuf.rs‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use crate::protobuf::{
44
};
55
use crate::{Metadata, Prettify, Reencode};
66
use anyhow::{Context, Result};
7+
use log::info;
78
use mitmproxy_highlight::Language;
89
use serde_yaml::Value;
910

@@ -37,8 +38,17 @@ impl Prettify for Protobuf {
3738
}
3839

3940
fn prettify(&self, data: &[u8], metadata: &dyn Metadata) -> Result<String> {
40-
let descriptor = existing_proto_definitions::find_best_match(metadata)?.unwrap_or_default();
41-
self.prettify_with_descriptor(data, &descriptor)
41+
let proto_def = existing_proto_definitions::find_best_match(metadata)?;
42+
if let Some(descriptor) = &proto_def {
43+
if let Ok(ret) = self.prettify_with_descriptor(data, descriptor) {
44+
return Ok(ret);
45+
}
46+
}
47+
let ret = self.prettify_with_descriptor(data, &DescriptorWithDeps::default())?;
48+
if proto_def.is_some() {
49+
info!("Existing protobuf definition does not match, parsing as unknown proto.");
50+
}
51+
Ok(ret)
4252
}
4353

4454
fn render_priority(&self, _data: &[u8], metadata: &dyn Metadata) -> f32 {
@@ -210,14 +220,16 @@ mod tests {
210220
assert_eq!(result, VARINT_PRETTY_YAML);
211221
}
212222

223+
/// When the existing proto definition does not match the wire data,
224+
/// but the wire data is still valid Protobuf, parse it raw.
213225
#[test]
214226
fn prettify_mismatch() {
215227
let metadata = TestMetadata::default().with_protobuf_definitions(concat!(
216228
env!("CARGO_MANIFEST_DIR"),
217229
"/testdata/protobuf/simple.proto"
218230
));
219-
let result = Protobuf.prettify(string::PROTO, &metadata);
220-
assert!(result.is_err());
231+
let result = Protobuf.prettify(string::PROTO, &metadata).unwrap();
232+
assert_eq!(result, string::YAML);
221233
}
222234

223235
#[test]

‎mitmproxy-contentviews/testdata/protobuf/mismatch.proto‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
syntax = "proto3";
2+
3+
message TestMessage {
4+
string example = 1;
5+
}

0 commit comments

Comments
 (0)