Skip to content

Commit c8aa28f

Browse files
authored
docs: Document Fn Execution API protos (#39293)
* Document Fn Execution API protos Add documentation comments to previously undocumented messages and fields in the Beam Fn API protos: beam_fn_api.proto: ProcessBundleResponse, ProcessBundleProgressResponse, FinalizeBundleRequest/Response, Elements.DrainMode, StateRequest/Response, BeamFnState service, StateKey, StateAppendResponse, StateClearRequest/Response, OrderedListRange, LogControl, StartWorkerRequest/Response, StopWorkerRequest/Response, BeamFnExternalWorkerPool service. beam_provision_api.proto: Consistency fix on dependencies field. * Feedback fixes
1 parent 1c9fd1b commit c8aa28f

2 files changed

Lines changed: 60 additions & 5 deletions

File tree

model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_fn_api.proto

Lines changed: 59 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ service BeamFnControl {
142142

143143
// Requests the ProcessBundleDescriptor with the given id.
144144
message GetProcessBundleDescriptorRequest {
145+
// (Required) The id of the ProcessBundleDescriptor to retrieve.
145146
string process_bundle_descriptor_id = 1;
146147
}
147148

@@ -440,6 +441,8 @@ message ProcessBundleRequest {
440441
string data_stream_id = 6;
441442
}
442443

444+
// The response to a ProcessBundleRequest, returned by the SDK harness after
445+
// bundle processing completes (or is partially completed with residual roots).
443446
message ProcessBundleResponse {
444447
// (Optional) Specifies that the bundle has not been completed and the
445448
// following applications need to be scheduled and executed in the future.
@@ -509,6 +512,8 @@ message MonitoringInfosMetadataRequest {
509512
repeated string monitoring_info_id = 1;
510513
}
511514

515+
// A response containing progress information for a currently active bundle.
516+
// This is returned in response to a ProcessBundleProgressRequest.
512517
message ProcessBundleProgressResponse {
513518
// DEPRECATED (Required) The list of metrics or other MonitoredState
514519
// collected while processing this bundle.
@@ -719,12 +724,17 @@ message ProcessBundleSplitResponse {
719724
}
720725

721726

727+
// A request sent by the runner to finalize a bundle that previously indicated
728+
// it requires finalization (via requires_finalization in ProcessBundleResponse).
729+
// This is sent after the runner has committed the output of the bundle, allowing
730+
// the SDK to perform any post-commit cleanup.
722731
message FinalizeBundleRequest {
723732
// (Required) A reference to a completed process bundle request with the given
724733
// instruction id.
725734
string instruction_id = 1;
726735
}
727736

737+
// A response to a FinalizeBundleRequest, indicating that finalization is complete.
728738
message FinalizeBundleResponse {
729739
// Empty
730740
}
@@ -763,10 +773,16 @@ message Elements {
763773
bool is_last = 4;
764774
}
765775

776+
// Indicates whether the bundle is being drained. When draining, sources
777+
// should stop producing new data and the SDK should finish processing
778+
// in-flight elements.
766779
message DrainMode {
767780
enum Enum {
781+
// Drain mode has not been specified.
768782
UNSPECIFIED = 0;
783+
// The bundle is not being drained; normal processing continues.
769784
NOT_DRAINING = 1;
785+
// The bundle is being drained; sources should stop producing new data.
770786
DRAINING = 2;
771787
}
772788
}
@@ -797,6 +813,8 @@ message Elements {
797813
// Element metadata passed as part of WindowedValue to make WindowedValue
798814
// extensible and backward compatible
799815
message ElementMetadata {
816+
// (Optional) Indicates whether the bundle is being drained, allowing the
817+
// SDK to handle drain semantics for this element.
800818
optional DrainMode.Enum drain = 1;
801819
// (Optional) As part of https://www.w3.org/TR/trace-context/ we are forwarding a trace and participating in it.
802820
// Traceparent header represents the incoming request in a tracing system in a common format.
@@ -869,6 +887,9 @@ service BeamFnData {
869887
* State API
870888
*/
871889

890+
// A request sent by the SDK harness to the runner to manipulated
891+
// state associated with a particular StateKey. State requests are scoped to
892+
// the instruction (bundle) that is currently being processed.
872893
message StateRequest {
873894
// (Required) A unique identifier provided by the SDK which represents this
874895
// requests execution. The StateResponse corresponding with this request
@@ -896,6 +917,8 @@ message StateRequest {
896917
}
897918
}
898919

920+
// A response from the runner to a StateRequest, containing the result of
921+
// the requested state operation.
899922
message StateResponse {
900923
// (Required) A reference provided by the SDK which represents a requests
901924
// execution. The StateResponse must have the matching id when responding
@@ -920,6 +943,10 @@ message StateResponse {
920943
}
921944
}
922945

946+
// An API for the SDK harness to access state stored by the runner on behalf
947+
// of the SDK. State is scoped to the currently processing bundle and can be
948+
// used for user state, side inputs, and runner-managed references.
949+
// Stable
923950
service BeamFnState {
924951
// Used to get/append/clear state stored by the runner on behalf of the SDK.
925952
//
@@ -934,6 +961,9 @@ service BeamFnState {
934961
stream StateResponse) {}
935962
}
936963

964+
// A key identifying which state to access. Exactly one of the oneof type
965+
// fields must be set, determining the kind of state being requested
966+
// (e.g. side input data, user state, or runner-managed references).
937967
message StateKey {
938968
message Runner {
939969
// (Required) Opaque information supplied by the runner. Used to support
@@ -1209,18 +1239,24 @@ message StateAppendRequest {
12091239
bytes data = 1;
12101240
}
12111241

1212-
// A response to append state.
1242+
// A response to a StateAppendRequest. Currently empty as append operations
1243+
// do not return data.
12131244
message StateAppendResponse {}
12141245

1215-
// A request to clear state.
1246+
// A request to clear all state associated with the given StateKey.
12161247
message StateClearRequest {}
12171248

1218-
// A response to clear state.
1249+
// A response to a StateClearRequest. Currently empty as clear operations
1250+
// do not return data.
12191251
message StateClearResponse {}
12201252

1221-
// A message describes a sort key range [start, end).
1253+
// A message describing a sort key range [start, end) for ordered list state.
1254+
// Used in OrderedListUserState to specify which portion of the ordered list
1255+
// to retrieve.
12221256
message OrderedListRange {
1257+
// (Required) The inclusive start of the sort key range.
12231258
int64 start = 1;
1259+
// (Required) The exclusive end of the sort key range.
12241260
int64 end = 2;
12251261
}
12261262

@@ -1314,6 +1350,8 @@ message LogEntry {
13141350
google.protobuf.Struct custom_data = 9;
13151351
}
13161352

1353+
// A control message sent from the runner to the SDK harness to configure
1354+
// logging behavior. Currently empty, reserved for future log-level control.
13171355
message LogControl {}
13181356

13191357
// Stable
@@ -1331,27 +1369,44 @@ service BeamFnLogging {
13311369
stream LogControl) {}
13321370
}
13331371

1372+
// A request from the runner to start a new SDK worker process with the given
1373+
// configuration, including endpoint addresses and parameters.
13341374
message StartWorkerRequest {
1375+
// (Required) A unique identifier for the worker to start.
13351376
string worker_id = 1;
1377+
// (Required) The control endpoint the worker should connect to.
13361378
org.apache.beam.model.pipeline.v1.ApiServiceDescriptor control_endpoint = 2;
1379+
// (Optional) The logging endpoint the worker should use.
13371380
org.apache.beam.model.pipeline.v1.ApiServiceDescriptor logging_endpoint = 3;
1381+
// (Optional) The artifact retrieval endpoint the worker should use.
13381382
org.apache.beam.model.pipeline.v1.ApiServiceDescriptor artifact_endpoint = 4;
1383+
// (Optional) The provisioning endpoint the worker should use.
13391384
org.apache.beam.model.pipeline.v1.ApiServiceDescriptor provision_endpoint = 5;
1385+
// (Optional) Additional runner-specific parameters to pass to the worker.
13401386
map<string, string> params = 10;
13411387
}
13421388

1389+
// A response from a StartWorkerRequest. If the worker failed to start,
1390+
// the error field will contain a human-readable error message.
13431391
message StartWorkerResponse {
1392+
// (Optional) Error message if the worker failed to start.
13441393
string error = 1;
13451394
}
13461395

1396+
// A request from the runner to stop a running SDK worker process.
13471397
message StopWorkerRequest {
1398+
// (Required) The unique identifier of the worker to stop.
13481399
string worker_id = 1;
13491400
}
13501401

13511402
message StopWorkerResponse {
1403+
// (Optional) Human-readable error message if the worker failed to stop cleanly.
13521404
string error = 1;
13531405
}
13541406

1407+
// An API for the runner to manage external SDK worker processes.
1408+
// Allows the runner to start and stop SDK worker instances, typically used
1409+
// for containerized or process-based execution environments.
13551410
service BeamFnExternalWorkerPool {
13561411
// Start the SDK worker with the given ID.
13571412
rpc StartWorker (StartWorkerRequest) returns (StartWorkerResponse) {}

model/fn-execution/src/main/proto/org/apache/beam/model/fn_execution/v1/beam_provision_api.proto

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ message ProvisionInfo {
7575
// (optional) The control endpoint this SDK should use.
7676
org.apache.beam.model.pipeline.v1.ApiServiceDescriptor control_endpoint = 10;
7777

78-
// The set of dependencies that should be staged into this environment.
78+
// (optional) The set of dependencies that should be staged into this environment.
7979
repeated org.apache.beam.model.pipeline.v1.ArtifactInformation dependencies = 11;
8080

8181
// (optional) A set of capabilities that this SDK is allowed to use in its

0 commit comments

Comments
 (0)