Document job management and interactive API protos (#39294)

Add documentation comments to previously undocumented messages and fields
in the Beam job management and interactive API protos:

beam_artifact_api.proto: ArtifactRetrievalService, ArtifactStagingService,
  ArtifactRequestWrapper, ProxyManifest, PutArtifactMetadata, and more.
beam_expansion_api.proto: ExpansionRequest/Response,
  DiscoverSchemaTransformRequest/Response, SchemaTransformConfig.
beam_job_api.proto: JobService, JobInfo, JobStateEvent, JobMessage,
  GetJobMetricsRequest/Response, MetricResults, PipelineOptionType.
beam_interactive_api.proto: TestStreamService, EventsRequest.

Author: damccorm

model/interactive/src/main/proto/org/apache/beam/model/interactive/v1/beam_interactive_api.proto

@@ -50,11 +50,15 @@ message TestStreamFileRecord {
   org.apache.beam.model.pipeline.v1.TestStreamPayload.Event recorded_event = 1;
 }
 
+// A gRPC service that serves cached TestStream events to a TestStream source.
+// Used by Interactive Beam to replay recorded pipeline elements from a streaming cache.
 service TestStreamService {
   // A TestStream will request for events using this RPC.
   rpc Events(EventsRequest) returns (stream org.apache.beam.model.pipeline.v1.TestStreamPayload.Event) {}
 }
 
+// A request for TestStream events from a streaming cache.
+// The client specifies which PCollection output tags to read events for.
 message EventsRequest {
   // The set of PCollections to read from. These are the PTransform outputs
   // local names. These are a subset of the TestStream's outputs. This allows

model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_artifact_api.proto

@@ -78,18 +78,23 @@ message ResolveArtifactsResponse {
 
 // A request to get an artifact.
 message GetArtifactRequest {
+  // (Required) The artifact to retrieve, typically a resolved artifact from ResolveArtifacts.
   org.apache.beam.model.pipeline.v1.ArtifactInformation artifact = 1;
 }
 
 // Part of a response to getting an artifact.
 message GetArtifactResponse {
+  // A chunk of the artifact's binary content.
   bytes data = 1;
 }
 
 // Wraps an ArtifactRetrievalService request for use in ReverseArtifactRetrievalService.
+// The server sends these requests to the client during the reverse staging flow.
 message ArtifactRequestWrapper {
   oneof request {
+    // Request to resolve a set of artifacts into concrete artifact references.
     ResolveArtifactsRequest resolve_artifact = 1000;
+    // Request to retrieve the bytes of a specific artifact.
     GetArtifactRequest get_artifact = 1001;
   }
 }
@@ -151,16 +156,22 @@ message ArtifactMetadata {
 
 // A collection of artifacts.
 message Manifest {
+  // (Required) The list of artifacts staged for a job.
   repeated ArtifactMetadata artifact = 1;
 }
 
 // A manifest with location information.
 message ProxyManifest {
+  // (Required) The manifest of staged artifacts.
   Manifest manifest = 1;
+  // A location (name + URI) at which a staged artifact can be retrieved.
   message Location {
-     string name = 1;
-     string uri = 2;
+    // (Required) The name of the artifact, matching the name in the Manifest.
+    string name = 1;
+    // (Required) The URI at which the artifact content can be retrieved.
+    string uri = 2;
   }
+  // (Required) The list of artifact locations for retrieval.
   repeated Location location = 2;
 }
 
@@ -173,6 +184,7 @@ message GetManifestRequest {
 
 // A response containing a job manifest.
 message GetManifestResponse {
+  // (Required) The manifest of staged artifacts for the job.
   Manifest manifest = 1;
 }
 
@@ -187,9 +199,11 @@ message LegacyGetArtifactRequest {
 
 // Part of an artifact.
 message ArtifactChunk {
+  // A chunk of the artifact's binary content.
   bytes data = 1;
 }
 
+// Metadata for an artifact being staged, sent as the first message in a PutArtifact stream.
 message PutArtifactMetadata {
   // (Required) A token for artifact staging session. This token can be obtained
   // from PrepareJob request in JobService
@@ -211,6 +225,7 @@ message PutArtifactRequest {
   }
 }
 
+// A response to staging an artifact. An empty response indicating the artifact was staged successfully.
 message PutArtifactResponse {
 }
 

model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_expansion_api.proto

@@ -33,6 +33,8 @@ import "google/protobuf/struct.proto";
 import "org/apache/beam/model/pipeline/v1/beam_runner_api.proto";
 import "org/apache/beam/model/pipeline/v1/schema.proto";
 
+// A request to expand a single PTransform in a remote SDK. The expansion service
+// resolves the transform into its constituent subtransforms and outputs.
 message ExpansionRequest {
   // Set of components needed to interpret the transform, or which
   // may be useful for its expansion.  This includes the input
@@ -64,6 +66,8 @@ message ExpansionRequest {
   google.protobuf.Struct pipeline_options = 6;
 }
 
+// The result of expanding a PTransform, containing the expanded transform and
+// all newly created components.
 message ExpansionResponse {
   // Set of components needed to execute the expanded transform,
   // including the (original) inputs, outputs, and subtransforms.
@@ -82,9 +86,13 @@ message ExpansionResponse {
   string error = 10;
 }
 
+// A request to discover all SchemaTransformProviders registered with the expansion service.
+// This is used by cross-language pipeline construction to enumerate available transforms.
 message DiscoverSchemaTransformRequest  {
 }
 
+// Configuration metadata for a SchemaTransform, describing its inputs, outputs,
+// and configuration schema. Used to enable cross-language transform discovery.
 message SchemaTransformConfig {
   // Config schema of the SchemaTransform
   org.apache.beam.model.pipeline.v1.Schema config_schema = 1;
@@ -102,6 +110,7 @@ message SchemaTransformConfig {
   string description = 4;
 }
 
+// A response containing all discovered SchemaTransform configurations.
 message DiscoverSchemaTransformResponse {
   // A mapping from SchemaTransform ID to schema transform config of discovered
   // SchemaTransforms

model/job-management/src/main/proto/org/apache/beam/model/job_management/v1/beam_job_api.proto

@@ -112,6 +112,7 @@ message RunJobRequest {
 }
 
 
+// The response from submitting a job for execution.
 message RunJobResponse {
   string job_id = 1; // (required) The ID for the executing job
 }
@@ -144,6 +145,7 @@ message DrainJobResponse {
 }
 
 // A subset of info provided by ProvisionApi.ProvisionInfo
+// Represents metadata about a submitted job.
 message JobInfo {
   string job_id = 1; // (required)
   string job_name = 2; // (required)
@@ -155,6 +157,7 @@ message JobInfo {
 // Throws error GRPC_STATUS_UNAVAILABLE if server is down
 message GetJobsRequest { }
 
+// The response from requesting a list of all invoked jobs.
 message GetJobsResponse {
   repeated JobInfo job_info = 1; // (required)
 }
@@ -168,6 +171,7 @@ message GetJobStateRequest {
 
 }
 
+// An event representing a job state transition. Emitted by GetState and GetStateStream.
 message JobStateEvent {
   JobState.Enum state = 1; // (required)
   google.protobuf.Timestamp timestamp = 2; // (required)
@@ -182,6 +186,7 @@ message GetJobPipelineRequest {
 
 }
 
+// The response from requesting a job's pipeline representation.
 message GetJobPipelineResponse {
   org.apache.beam.model.pipeline.v1.Pipeline pipeline = 1; // (required)
 }
@@ -195,25 +200,41 @@ message JobMessagesRequest {
   string job_id = 1; // (required)
 }
 
+// A single log message or diagnostic event from a running job.
 message JobMessage {
+  // (Required) A unique identifier for this message.
   string message_id = 1;
+  // (Required) The time at which the message was emitted, as a string representation.
   string time = 2;
+  // (Required) The importance level of this message.
   MessageImportance importance = 3;
+  // (Required) The text content of the message.
   string message_text = 4;
 
+  // Importance levels for job messages, ordered from least to most severe.
   enum MessageImportance {
+    // The importance was not specified.
     MESSAGE_IMPORTANCE_UNSPECIFIED = 0;
+    // Debug-level messages, typically very verbose.
     JOB_MESSAGE_DEBUG = 1;
+    // Detailed informational messages.
     JOB_MESSAGE_DETAILED = 2;
+    // Basic informational messages.
     JOB_MESSAGE_BASIC = 3;
+    // Warning messages indicating potential issues.
     JOB_MESSAGE_WARNING = 4;
+    // Error messages indicating failures.
     JOB_MESSAGE_ERROR = 5;
   }
 }
 
+// A streaming response from GetMessageStream, containing either a job message
+// or a job state change event.
 message JobMessagesResponse {
   oneof response {
+    // A log message or diagnostic event from the job.
     JobMessage message_response = 1;
+    // A job state transition event.
     JobStateEvent state_response = 2;
   }
 }
@@ -270,17 +291,22 @@ message JobState {
 }
 
 
+// A request to fetch metrics for a given job.
 message GetJobMetricsRequest {
   string job_id = 1; // (required)
 }
 
+// A response containing metrics for a given job.
 message GetJobMetricsResponse {
+  // (Required) The metrics results containing both attempted and committed values.
   MetricResults metrics = 1;
 }
 
 // All metrics for a given job.  Runners may support one or the other or both.
 message MetricResults {
+  // Metrics reflecting the result of attempted (non-committed) computations.
   repeated org.apache.beam.model.pipeline.v1.MonitoringInfo attempted = 1;
+  // Metrics reflecting the result of committed computations.
   repeated org.apache.beam.model.pipeline.v1.MonitoringInfo committed = 2;
 }
 
@@ -296,12 +322,17 @@ message DescribePipelineOptionsRequest {
 // Types mirror those of JSON, since that's how pipeline options are serialized.
 message PipelineOptionType {
   enum Enum {
+    // A string value.
     STRING = 0;
+    // A boolean (true/false) value.
     BOOLEAN = 1;
     // whole numbers, see https://json-schema.org/understanding-json-schema/reference/numeric.html
     INTEGER = 2;
+    // A floating-point number.
     NUMBER = 3;
+    // An array of values.
     ARRAY = 4;
+    // A nested object.
     OBJECT = 5;
   };
 }
@@ -324,6 +355,7 @@ message PipelineOptionDescriptor {
   string group = 5;
 }
 
+// The response from describing pipeline options, containing a list of option descriptors.
 message DescribePipelineOptionsResponse {
   // List of pipeline option descriptors.
   repeated PipelineOptionDescriptor options = 1;