Skip to content

Commit 235d6b8

Browse files
authored
Feature/no descriptor fallback (#84)
* generate descriptor-less variants of test services * add packed field variant to proto2 features * create a synthesizer to generate descriptors, testing against real descriptor values * wire in synthesizer as a fallback option * review corner cases and increase coverage * update README * corner case checking * synthesize deprecated flag too, its available * add some actually-reflecting integration tests * reduce test duplication where it doesn't add anything * linting * ensure loaded even when synthesizing
1 parent 078994a commit 235d6b8

42 files changed

Lines changed: 3034 additions & 32 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

lib/grpc_reflection.ex

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@ defmodule GrpcReflection do
22
@moduledoc """
33
Reflection support for the grpc-elixir package
44
5-
To use these servers, all protos must be compiled with the `gen_descriptors=true` option, as that is the source of truth for the reflection service.
5+
Protos compiled with `gen_descriptors=true` provide the richest reflection output.
6+
If that option is omitted, this library synthesizes descriptors from runtime module
7+
metadata (`__message_props__`, `__rpc_calls__`). The synthesized path produces
8+
equivalent output for standard gRPC reflection clients; only proto2 extensions and
9+
custom proto options are unavailable without `gen_descriptors=true`.
610
711
To turn on reflection in your application, do the following:
812

lib/grpc_reflection/service/builder.ex

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ defmodule GrpcReflection.Service.Builder do
6969
end
7070

7171
defp trace_message_refs(state, parent_symbol, module) do
72-
case module.descriptor() do
72+
case get_descriptor(module) do
7373
%{field: fields} ->
7474
trace_message_fields(state, parent_symbol, module, fields)
7575

@@ -79,7 +79,7 @@ defmodule GrpcReflection.Service.Builder do
7979
end
8080

8181
defp trace_message_fields(state, parent_symbol, module, fields) do
82-
nested_types = Util.get_nested_types(parent_symbol, module.descriptor())
82+
nested_types = Util.get_nested_types(parent_symbol, get_descriptor(module))
8383

8484
module.__message_props__().field_props
8585
|> Map.values()
@@ -176,9 +176,37 @@ defmodule GrpcReflection.Service.Builder do
176176
# generate descriptors. Use this to potentially unwrap the service proto when dealing
177177
# with descriptors that could come from a service module.
178178
defp get_descriptor(module) do
179-
case module.descriptor() do
180-
%FileDescriptorProto{service: [proto]} -> proto
181-
proto -> proto
179+
cond do
180+
not Code.ensure_loaded?(module) ->
181+
raise "Module #{inspect(module)} is not loaded"
182+
183+
function_exported?(module, :descriptor, 0) ->
184+
case module.descriptor() do
185+
%FileDescriptorProto{service: [proto]} -> proto
186+
proto -> proto
187+
end
188+
189+
true ->
190+
synthesize_descriptor(module)
191+
end
192+
end
193+
194+
defp synthesize_descriptor(module) do
195+
cond do
196+
function_exported?(module, :__rpc_calls__, 0) ->
197+
GrpcReflection.Service.Builder.Synthesizer.service_descriptor(module)
198+
199+
function_exported?(module, :mapping, 0) ->
200+
GrpcReflection.Service.Builder.Synthesizer.enum_descriptor(module)
201+
202+
function_exported?(module, :__message_props__, 0) ->
203+
GrpcReflection.Service.Builder.Synthesizer.message_descriptor(module)
204+
205+
true ->
206+
# unreachable in practice: validate_services/1 only admits modules that export
207+
# __rpc_calls__/0, __message_props__/0, or descriptor/0, so a module reaching
208+
# synthesize_descriptor/1 without any of those three is impossible.
209+
raise "Module #{inspect(module)} exports neither descriptor/0, __rpc_calls__/0, nor __message_props__/0 — cannot synthesize a descriptor"
182210
end
183211
end
184212
end

lib/grpc_reflection/service/builder/extensions.ex

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,20 @@ defmodule GrpcReflection.Service.Builder.Extensions do
55
alias GrpcReflection.Service.State
66

77
def add_extensions(state, symbol, module) do
8-
extension_file = symbol <> "Extension.proto"
8+
if Code.ensure_loaded?(module) and function_exported?(module, :descriptor, 0) do
9+
extension_file = symbol <> "Extension.proto"
910

10-
case process_extensions(module, symbol, extension_file, module.descriptor()) do
11-
{:ok, {extension_numbers, extension_payload}} ->
12-
state
13-
|> State.add_file(extension_payload)
14-
|> State.add_extensions(%{symbol => extension_numbers})
11+
case process_extensions(module, symbol, extension_file, module.descriptor()) do
12+
{:ok, {extension_numbers, extension_payload}} ->
13+
state
14+
|> State.add_file(extension_payload)
15+
|> State.add_extensions(%{symbol => extension_numbers})
1516

16-
:ignore ->
17-
state
17+
:ignore ->
18+
state
19+
end
20+
else
21+
state
1822
end
1923
end
2024

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
defmodule GrpcReflection.Service.Builder.Synthesizer do
2+
@moduledoc false
3+
4+
# Builds a FieldDescriptorProto from a Protobuf.FieldProps.
5+
# Note: oneof_index is NOT set here for proto3_optional fields — those synthetic
6+
# oneof indices are assigned in message_descriptor/1 where all fields are visible.
7+
# syntax must be passed explicitly when building from message context; proto3 repeated
8+
# scalars have packed?: true in FieldProps but must NOT set options.packed (implicit).
9+
def field_descriptor_from_props(%Protobuf.FieldProps{} = props, syntax \\ :proto3) do
10+
{type, type_name} = resolve_type(props)
11+
12+
%Google.Protobuf.FieldDescriptorProto{
13+
name: props.name,
14+
number: props.fnum,
15+
json_name: props.json_name,
16+
type: type,
17+
type_name: type_name,
18+
label: resolve_label(props),
19+
default_value: encode_default(props.default),
20+
oneof_index: props.oneof,
21+
options: field_options(props, syntax),
22+
proto3_optional: props.proto3_optional?,
23+
extendee: nil
24+
}
25+
end
26+
27+
def message_descriptor(module) do
28+
props = module.__message_props__()
29+
30+
fields =
31+
props.ordered_tags
32+
|> Enum.map(&props.field_props[&1])
33+
|> Enum.map(&field_descriptor_from_props(&1, props.syntax))
34+
|> assign_proto3_optional_oneof_indices(length(props.oneof))
35+
36+
real_oneof_decls =
37+
Enum.map(props.oneof, fn {name, _tag} ->
38+
%Google.Protobuf.OneofDescriptorProto{name: Atom.to_string(name)}
39+
end)
40+
41+
synthetic_oneof_decls =
42+
fields
43+
|> Enum.filter(& &1.proto3_optional)
44+
|> Enum.map(fn f ->
45+
%Google.Protobuf.OneofDescriptorProto{name: "_#{f.name}"}
46+
end)
47+
48+
extension_ranges =
49+
(props.extension_range || [])
50+
|> Enum.map(fn {start, stop} ->
51+
%Google.Protobuf.DescriptorProto.ExtensionRange{start: start, end: stop}
52+
end)
53+
54+
short_name = module.full_name() |> String.split(".") |> List.last()
55+
56+
%Google.Protobuf.DescriptorProto{
57+
name: short_name,
58+
field: fields,
59+
oneof_decl: real_oneof_decls ++ synthetic_oneof_decls,
60+
extension_range: extension_ranges,
61+
# nested_type is always [] in synthesized descriptors. All message types — including
62+
# map-entry types and any inline nested message definitions in the original .proto —
63+
# are discovered via field traversal in builder.ex and emitted as separate files.
64+
# This differs from protoc output (which embeds nested types in the parent descriptor)
65+
# but is handled correctly by standard reflection clients.
66+
nested_type: [],
67+
enum_type: []
68+
}
69+
end
70+
71+
def enum_descriptor(module) do
72+
short_name = module.full_name() |> String.split(".") |> List.last()
73+
74+
# mapping() is a map so has no intrinsic order; __message_props__.ordered_tags is
75+
# numerically sorted. Neither preserves proto declaration order, so we sort by value.
76+
# This differs from protoc output for enums with non-monotonic declaration order
77+
# (e.g. negative sentinel values declared after positive ones), but reflection
78+
# clients look up enum values by number, not position, so this is safe in practice.
79+
values =
80+
module.__message_props__().ordered_tags
81+
|> Enum.map(fn tag ->
82+
props = module.__message_props__().field_props[tag]
83+
%Google.Protobuf.EnumValueDescriptorProto{name: props.name, number: props.fnum}
84+
end)
85+
86+
%Google.Protobuf.EnumDescriptorProto{name: short_name, value: values}
87+
end
88+
89+
def service_descriptor(module) do
90+
service_name = module.__meta__(:name) |> String.split(".") |> List.last()
91+
92+
methods =
93+
module.__rpc_calls__()
94+
|> Enum.map(fn
95+
{method, {req, req_stream}, {resp, resp_stream}} ->
96+
build_method_descriptor(method, req, req_stream, resp, resp_stream)
97+
98+
{method, {req, req_stream}, {resp, resp_stream}, _opts} ->
99+
build_method_descriptor(method, req, req_stream, resp, resp_stream)
100+
end)
101+
102+
%Google.Protobuf.ServiceDescriptorProto{name: service_name, method: methods}
103+
end
104+
105+
defp resolve_label(%Protobuf.FieldProps{required?: true}), do: :LABEL_REQUIRED
106+
defp resolve_label(%Protobuf.FieldProps{repeated?: true}), do: :LABEL_REPEATED
107+
defp resolve_label(%Protobuf.FieldProps{map?: true}), do: :LABEL_REPEATED
108+
defp resolve_label(_), do: :LABEL_OPTIONAL
109+
110+
defp resolve_type(%Protobuf.FieldProps{enum?: true, type: {:enum, mod}}) do
111+
{:TYPE_ENUM, "." <> mod.full_name()}
112+
end
113+
114+
defp resolve_type(%Protobuf.FieldProps{embedded?: true, type: mod}) when is_atom(mod) do
115+
{:TYPE_MESSAGE, "." <> mod.full_name()}
116+
end
117+
118+
defp resolve_type(%Protobuf.FieldProps{type: type}) do
119+
{:"TYPE_#{type |> Atom.to_string() |> String.upcase()}", nil}
120+
end
121+
122+
# proto3 repeated scalars are implicitly packed — reflection clients infer this from
123+
# syntax, so options.packed must not be set (matches real protoc output).
124+
defp field_options(props, syntax) do
125+
packed = props.packed? == true and syntax == :proto2
126+
deprecated = props.deprecated? == true
127+
128+
case {packed, deprecated} do
129+
{false, false} ->
130+
nil
131+
132+
{packed, deprecated} ->
133+
%Google.Protobuf.FieldOptions{packed: packed || nil, deprecated: deprecated || nil}
134+
end
135+
end
136+
137+
defp encode_default(nil), do: nil
138+
defp encode_default(v) when is_binary(v), do: v
139+
defp encode_default(v) when is_boolean(v), do: to_string(v)
140+
defp encode_default(v) when is_integer(v), do: Integer.to_string(v)
141+
defp encode_default(v) when is_float(v), do: Float.to_string(v)
142+
defp encode_default(v) when is_atom(v), do: Atom.to_string(v)
143+
144+
# proto3_optional fields have no oneof in __message_props__ (the runtime doesn't need
145+
# the synthetic oneof) but the wire format and reflection clients expect oneof_index to
146+
# point at a synthetic "_fieldname" oneof entry appended after any real oneofs.
147+
defp assign_proto3_optional_oneof_indices(fields, real_oneof_count) do
148+
fields
149+
|> Enum.map_reduce(0, fn field, counter ->
150+
if field.proto3_optional do
151+
{%{field | oneof_index: real_oneof_count + counter}, counter + 1}
152+
else
153+
{field, counter}
154+
end
155+
end)
156+
|> elem(0)
157+
end
158+
159+
defp build_method_descriptor(method, req, req_stream, resp, resp_stream) do
160+
%Google.Protobuf.MethodDescriptorProto{
161+
name: Atom.to_string(method),
162+
input_type: "." <> req.full_name(),
163+
output_type: "." <> resp.full_name(),
164+
client_streaming: req_stream,
165+
server_streaming: resp_stream
166+
}
167+
end
168+
end

lib/grpc_reflection/service/builder/util.ex

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,11 @@ defmodule GrpcReflection.Service.Builder.Util do
107107
def validate_services(services) do
108108
invalid_services =
109109
Enum.reject(services, fn service_mod ->
110-
is_binary(service_mod.__meta__(:name)) and
111-
is_struct(service_mod.descriptor())
110+
Code.ensure_loaded?(service_mod) and
111+
function_exported?(service_mod, :__meta__, 1) and
112+
is_binary(service_mod.__meta__(:name)) and
113+
(function_exported?(service_mod, :descriptor, 0) or
114+
function_exported?(service_mod, :__rpc_calls__, 0))
112115
end)
113116

114117
case invalid_services do

mix.exs

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ defmodule GrpcReflection.MixProject do
3838
~r/^PackageB\./,
3939
~r/^NestedEnumConflict\./,
4040
~r/^RecursiveMessage\./,
41+
~r/^NoDescriptor\./,
4142
GrpcReflection.TestEndpoint,
4243
GrpcReflection.TestEndpoint.Endpoint
4344
]
@@ -74,16 +75,15 @@ defmodule GrpcReflection.MixProject do
7475
]
7576
end
7677

77-
defp build_protos(_argv) do
78-
options =
79-
Enum.join(
80-
[
81-
"gen_descriptors=true",
82-
"plugins=grpc"
83-
],
84-
","
85-
)
78+
@protoc_opts "gen_descriptors=true,plugins=grpc"
79+
@protoc_opts_no_descriptor "package_prefix=NoDescriptor,plugins=grpc"
80+
# Protos that set (elixirpb.file).module_prefix must be skipped here: that option
81+
# overrides package_prefix entirely, so the no-descriptor pass would emit modules with
82+
# the same name as the descriptor pass, causing a compile-time conflict.
83+
# Add any proto that uses the elixirpb.file module_prefix option to this list.
84+
@skip_no_descriptor ["custom_prefix_service.proto"]
8685

86+
defp build_protos(_argv) do
8787
# compile reflection protos
8888
Enum.each(
8989
[
@@ -92,19 +92,25 @@ defmodule GrpcReflection.MixProject do
9292
],
9393
fn reflection_proto ->
9494
Mix.shell().cmd(
95-
"protoc --elixir_out=#{options}:./lib/proto --proto_path=priv/protos/ #{reflection_proto}"
95+
"protoc --elixir_out=#{@protoc_opts}:./lib/proto --proto_path=priv/protos/ #{reflection_proto}"
9696
)
9797
end
9898
)
9999

100-
# compile test protos
100+
# compile test protos — once with descriptors, once without
101101
"./priv/protos"
102102
|> File.ls!()
103103
|> Enum.filter(&Regex.match?(~r/.*.proto$/, &1))
104-
|> Enum.each(fn reflection_proto ->
104+
|> Enum.each(fn proto ->
105105
Mix.shell().cmd(
106-
"protoc --elixir_out=#{options}:./test/support/protos -I priv/protos/ -I deps/protobuf/src #{reflection_proto}"
106+
"protoc --elixir_out=#{@protoc_opts}:./test/support/protos -I priv/protos/ -I deps/protobuf/src #{proto}"
107107
)
108+
109+
unless proto in @skip_no_descriptor do
110+
Mix.shell().cmd(
111+
"protoc --elixir_out=#{@protoc_opts_no_descriptor}:./test/support/protos/no_descriptor -I priv/protos/ -I deps/protobuf/src #{proto}"
112+
)
113+
end
108114
end)
109115
end
110116

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
syntax = "proto3";
2+
3+
package deprecated_fields;
4+
5+
option go_package = "deprecated_fields";
6+
7+
// A message with a mix of active and deprecated fields.
8+
message DeprecatedRequest {
9+
string active_field = 1;
10+
string legacy_field = 2 [deprecated = true];
11+
int32 active_id = 3;
12+
int32 old_id = 4 [deprecated = true];
13+
}
14+
15+
message DeprecatedResponse {
16+
bool success = 1;
17+
string result = 2;
18+
string old_result = 3 [deprecated = true];
19+
}
20+
21+
service DeprecatedFieldsService {
22+
rpc Process(DeprecatedRequest) returns (DeprecatedResponse);
23+
}

priv/protos/proto2_features.proto

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ message Proto2Request {
3737
// Repeated Any in proto2
3838
repeated google.protobuf.Any any_values = 15;
3939

40+
// Packed repeated field (proto2 requires explicit [packed=true])
41+
repeated int32 packed_ints = 16 [packed=true];
42+
4043
// Extensions range
4144
extensions 100 to 199;
4245
}

0 commit comments

Comments
 (0)