Skip to content

Commit de91d4f

Browse files
zhumin8zhumin8
authored
feat(internal/librarian/java): support showcase as source in generate (#5799)
Support showcase as source in generate, pass both showcase dir and googleapis dir to protoc via -I flags, use only showcase dir for copy protos, and finding service configs. Also renaming variables related to googleapisDir for clarity. For #5731
1 parent f8a6cc4 commit de91d4f

9 files changed

Lines changed: 180 additions & 93 deletions

File tree

‎internal/librarian/java/generate.go‎

Lines changed: 69 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -56,31 +56,36 @@ func Generate(ctx context.Context, cfg *config.Config, library *config.Library,
5656
if err != nil {
5757
return fmt.Errorf("failed to resolve output directory path: %w", err)
5858
}
59-
// Ensure googleapisDir is absolute to avoid issues with relative paths in protoc.
60-
var googleapisDir string
61-
googleapisDir, err = filepath.Abs(srcs.Googleapis)
62-
if err != nil {
63-
return fmt.Errorf("failed to resolve googleapis directory path: %w", err)
64-
}
6559
if err := os.MkdirAll(outdir, 0755); err != nil {
6660
return fmt.Errorf("failed to create output directory %q: %w", outdir, err)
6761
}
62+
srcCfg := sources.NewSourceConfig(srcs, library.Roots)
63+
primaryDir := srcCfg.Root(srcCfg.ActiveRoots[0])
64+
6865
// generate repo metadata prior to client because info is needed for
6966
// owlbot.py to generate README.md
70-
metadata, err := generateRepoMetadata(cfg, library, outdir, googleapisDir)
67+
metadata, err := generateRepoMetadata(cfg, library, outdir, primaryDir)
7168
if err != nil {
7269
return fmt.Errorf("failed to generate .repo-metadata.json: %w", err)
7370
}
7471

7572
transports := make(map[string]serviceconfig.Transport)
7673
for _, api := range library.APIs {
77-
apiCfg, err := serviceconfig.Find(googleapisDir, api.Path, config.LanguageJava)
74+
apiCfg, err := serviceconfig.Find(primaryDir, api.Path, config.LanguageJava)
7875
if err != nil {
7976
return fmt.Errorf("failed to find api config for %s: %w", api.Path, err)
8077
}
8178
transports[api.Path] = apiCfg.Transport(config.LanguageJava)
8279
// metadata is needed for pom.xml generation in post process
83-
if err := generateAPI(ctx, cfg, api, library, googleapisDir, outdir, metadata, apiCfg); err != nil {
80+
if err := generateAPI(ctx, generateAPIParams{
81+
cfg: cfg,
82+
api: api,
83+
library: library,
84+
srcCfg: srcCfg,
85+
outdir: outdir,
86+
metadata: metadata,
87+
apiCfg: apiCfg,
88+
}); err != nil {
8489
return fmt.Errorf("failed to generate api %q: %w", api.Path, err)
8590
}
8691
}
@@ -94,7 +99,6 @@ func Generate(ctx context.Context, cfg *config.Config, library *config.Library,
9499
}); err != nil {
95100
return err
96101
}
97-
98102
return nil
99103
}
100104

@@ -107,63 +111,76 @@ func deriveAPIBase(library *config.Library, apiPath string) string {
107111
return path.Base(apiPath)
108112
}
109113

110-
func generateAPI(ctx context.Context, cfg *config.Config, api *config.API, library *config.Library, googleapisDir, outdir string, metadata *repoMetadata, apiCfg *serviceconfig.API) error {
111-
javaAPI := ResolveJavaAPI(library, api)
112-
p := postProcessParams{
113-
cfg: cfg,
114-
library: library,
114+
type generateAPIParams struct {
115+
cfg *config.Config
116+
api *config.API
117+
library *config.Library
118+
srcCfg *sources.SourceConfig
119+
outdir string
120+
metadata *repoMetadata
121+
apiCfg *serviceconfig.API
122+
}
123+
124+
func generateAPI(ctx context.Context, params generateAPIParams) error {
125+
javaAPI := ResolveJavaAPI(params.library, params.api)
126+
primaryDir := params.srcCfg.Root(params.srcCfg.ActiveRoots[0])
127+
googleapisDir := params.srcCfg.Root("googleapis")
128+
129+
postParams := postProcessParams{
130+
cfg: params.cfg,
131+
library: params.library,
115132
javaAPI: javaAPI,
116-
metadata: metadata,
117-
outDir: outdir,
118-
apiBase: deriveAPIBase(library, api.Path),
119-
googleapisDir: googleapisDir,
133+
metadata: params.metadata,
134+
outDir: params.outdir,
135+
apiBase: deriveAPIBase(params.library, params.api.Path),
136+
protoSourceDir: primaryDir,
120137
includeSamples: *javaAPI.Samples,
121138
}
122-
gapicDir := p.gapicDir()
123-
gRPCDir := p.gRPCDir()
124-
protoDir := p.protoDir()
139+
gapicDir := postParams.gapicDir()
140+
gRPCDir := postParams.gRPCDir()
141+
protoDir := postParams.protoDir()
125142
for _, dir := range []string{gapicDir, gRPCDir, protoDir} {
126143
if err := os.MkdirAll(dir, 0755); err != nil {
127144
return fmt.Errorf("failed to create directory %q: %w", dir, err)
128145
}
129146
}
130147

131-
apiDir := filepath.Join(googleapisDir, api.Path)
132-
apiProtos, err := gatherProtos(apiDir, api.Path)
148+
apiDir := filepath.Join(primaryDir, params.api.Path)
149+
apiProtos, err := gatherProtos(apiDir, params.api.Path)
133150
if err != nil {
134151
return fmt.Errorf("failed to find protos: %w", err)
135152
}
136-
apiProtos = filterProtos(apiProtos, javaAPI.ExcludedProtos, googleapisDir)
153+
apiProtos = filterProtos(apiProtos, javaAPI.ExcludedProtos, primaryDir)
137154
if len(apiProtos) == 0 {
138-
return fmt.Errorf("%s: %w", api.Path, errNoProtos)
155+
return fmt.Errorf("%s: %w", params.api.Path, errNoProtos)
139156
}
140-
p.apiProtos = apiProtos
157+
postParams.apiProtos = apiProtos
141158

142159
// 1. Generate standard Protocol Buffer Java classes.
143-
protoProtos := filterProtos(apiProtos, javaAPI.SkipProtoClassGeneration, googleapisDir)
144-
if err := runProtoc(ctx, protoProtocArgs(protoProtos, googleapisDir, protoDir)); err != nil {
160+
protoProtos := filterProtos(apiProtos, javaAPI.SkipProtoClassGeneration, primaryDir)
161+
if err := runProtoc(ctx, protoProtocArgs(protoProtos, params.srcCfg, protoDir)); err != nil {
145162
return fmt.Errorf("failed to generate proto: %w", err)
146163
}
147164
// 2. Generate gRPC service stubs (skipped if transport is rest).
148-
transport := apiCfg.Transport(config.LanguageJava)
165+
transport := params.apiCfg.Transport(config.LanguageJava)
149166
if transport != "rest" {
150-
if err := runProtoc(ctx, gRPCProtocArgs(apiProtos, googleapisDir, gRPCDir)); err != nil {
167+
if err := runProtoc(ctx, gRPCProtocArgs(apiProtos, params.srcCfg, gRPCDir)); err != nil {
151168
return fmt.Errorf("failed to generate gRPC module: %w", err)
152169
}
153170
}
154171
// 3. Generate GAPIC library.
155172
if !javaAPI.ProtoGRPCOnly {
156-
gapicOpts, err := resolveGAPICOptions(cfg, library, api, googleapisDir, apiCfg)
173+
gapicOpts, err := resolveGAPICOptions(params.cfg, params.library, params.api, primaryDir, params.apiCfg)
157174
if err != nil {
158175
return fmt.Errorf("failed to resolve gapic options: %w", err)
159176
}
160177
additionalProtos := deriveAdditionalProtoPaths(javaAPI, googleapisDir)
161-
if err := runProtoc(ctx, gapicProtocArgs(apiProtos, additionalProtos, googleapisDir, gapicDir, gapicOpts)); err != nil {
178+
if err := runProtoc(ctx, gapicProtocArgs(apiProtos, additionalProtos, params.srcCfg, gapicDir, gapicOpts)); err != nil {
162179
return fmt.Errorf("failed to generate gapic: %w", err)
163180
}
164181
}
165182

166-
if err := postProcessAPI(ctx, p); err != nil {
183+
if err := postProcessAPI(ctx, postParams); err != nil {
167184
return fmt.Errorf("failed to post process: %w", err)
168185
}
169186
return nil
@@ -190,37 +207,40 @@ var runProtoc = func(ctx context.Context, args []string) error {
190207
return command.Run(ctx, "protoc", args...)
191208
}
192209

193-
func baseProtocArgs(googleapisDir string) []string {
194-
return []string{
210+
func baseProtocArgs(srcCfg *sources.SourceConfig) []string {
211+
args := []string{
195212
"--experimental_allow_proto3_optional",
196-
"-I=" + googleapisDir,
197213
}
214+
for _, root := range srcCfg.ActiveRoots {
215+
args = append(args, "-I="+srcCfg.Root(root))
216+
}
217+
return args
198218
}
199219

200-
func protoProtocArgs(apiProtos []string, googleapisDir, protoDir string) []string {
201-
args := baseProtocArgs(googleapisDir)
220+
func protoProtocArgs(apiProtos []string, srcCfg *sources.SourceConfig, protoDir string) []string {
221+
args := baseProtocArgs(srcCfg)
202222
args = append(args, fmt.Sprintf("--java_out=%s", protoDir))
203223
args = append(args, apiProtos...)
204224
return args
205225
}
206226

207-
func gRPCProtocArgs(apiProtos []string, googleapisDir, gRPCDir string) []string {
208-
args := baseProtocArgs(googleapisDir)
227+
func gRPCProtocArgs(apiProtos []string, srcCfg *sources.SourceConfig, gRPCDir string) []string {
228+
args := baseProtocArgs(srcCfg)
209229
args = append(args, fmt.Sprintf("--java_grpc_out=%s", gRPCDir))
210230
args = append(args, apiProtos...)
211231
return args
212232
}
213233

214-
func gapicProtocArgs(apiProtos, additionalProtos []string, googleapisDir, gapicDir string, gapicOpts []string) []string {
215-
args := baseProtocArgs(googleapisDir)
234+
func gapicProtocArgs(apiProtos, additionalProtos []string, srcCfg *sources.SourceConfig, gapicDir string, gapicOpts []string) []string {
235+
args := baseProtocArgs(srcCfg)
216236
args = append(args, fmt.Sprintf("--java_gapic_out=metadata:%s", gapicDir))
217237
args = append(args, "--java_gapic_opt="+strings.Join(gapicOpts, ","))
218238
args = append(args, apiProtos...)
219239
args = append(args, additionalProtos...)
220240
return args
221241
}
222242

223-
func resolveGAPICOptions(cfg *config.Config, library *config.Library, api *config.API, googleapisDir string, apiCfg *serviceconfig.API) ([]string, error) {
243+
func resolveGAPICOptions(cfg *config.Config, library *config.Library, api *config.API, sourceDir string, apiCfg *serviceconfig.API) ([]string, error) {
224244
// gapicOpts are passed to the GAPIC generator via --java_gapic_opt.
225245
// "metadata" enables the generation of gapic_metadata.json and GraalVM reflect-config.json.
226246
gapicOpts := []string{"metadata"}
@@ -231,26 +251,26 @@ func resolveGAPICOptions(cfg *config.Config, library *config.Library, api *confi
231251
if apiCfg.ServiceConfig != "" {
232252
// api-service-config specifies the service YAML (e.g., logging_v2.yaml) which
233253
// contains documentation, HTTP rules, and other API-level configuration.
234-
gapicOpts = append(gapicOpts, gapicOpt("api-service-config", filepath.Join(googleapisDir, apiCfg.ServiceConfig)))
254+
gapicOpts = append(gapicOpts, gapicOpt("api-service-config", filepath.Join(sourceDir, apiCfg.ServiceConfig)))
235255
}
236256

237-
gapicConfig, err := serviceconfig.FindGAPICConfig(googleapisDir, api.Path)
257+
gapicConfig, err := serviceconfig.FindGAPICConfig(sourceDir, api.Path)
238258
if err != nil {
239259
return nil, fmt.Errorf("failed to find gapic config: %w", err)
240260
}
241261
if gapicConfig != "" {
242262
// gapic-config specifies the GAPIC configuration (e.g., logging_gapic.yaml) which
243263
// contains batching, LRO retries, and language settings.
244-
gapicOpts = append(gapicOpts, gapicOpt("gapic-config", filepath.Join(googleapisDir, gapicConfig)))
264+
gapicOpts = append(gapicOpts, gapicOpt("gapic-config", filepath.Join(sourceDir, gapicConfig)))
245265
}
246266

247-
gRPCServiceConfig, err := serviceconfig.FindGRPCServiceConfig(googleapisDir, api.Path)
267+
gRPCServiceConfig, err := serviceconfig.FindGRPCServiceConfig(sourceDir, api.Path)
248268
if err != nil {
249269
return nil, fmt.Errorf("failed to find gRPC service config: %w", err)
250270
}
251271
if gRPCServiceConfig != "" {
252272
// grpc-service-config specifies the retry and timeout settings for the gRPC client.
253-
gapicOpts = append(gapicOpts, gapicOpt("grpc-service-config", filepath.Join(googleapisDir, gRPCServiceConfig)))
273+
gapicOpts = append(gapicOpts, gapicOpt("grpc-service-config", filepath.Join(sourceDir, gRPCServiceConfig)))
254274
}
255275

256276
// transport specifies whether to generate gRPC, REST, or both types of clients.

‎internal/librarian/java/generate_test.go‎

Lines changed: 36 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,8 @@ func TestProtoProtocArgs(t *testing.T) {
179179
filepath.Join(googleapisDir, "google/cloud/secretmanager/v1/resources.proto"),
180180
filepath.Join(googleapisDir, "google/cloud/secretmanager/v1/service.proto"),
181181
}
182-
got := protoProtocArgs(apiProtos, googleapisDir, "proto-out")
182+
srcCfg := sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil)
183+
got := protoProtocArgs(apiProtos, srcCfg, "proto-out")
183184
want := []string{
184185
"--experimental_allow_proto3_optional",
185186
"-I=" + googleapisDir,
@@ -197,7 +198,8 @@ func TestGRPCProtocArgs(t *testing.T) {
197198
filepath.Join(googleapisDir, "google/cloud/secretmanager/v1/resources.proto"),
198199
filepath.Join(googleapisDir, "google/cloud/secretmanager/v1/service.proto"),
199200
}
200-
got := gRPCProtocArgs(apiProtos, googleapisDir, "grpc-out")
201+
srcCfg := sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil)
202+
got := gRPCProtocArgs(apiProtos, srcCfg, "grpc-out")
201203
want := []string{
202204
"--experimental_allow_proto3_optional",
203205
"-I=" + googleapisDir,
@@ -218,7 +220,8 @@ func TestGAPICProtocArgs(t *testing.T) {
218220
additionalProtos := []string{
219221
filepath.Join(googleapisDir, "google/cloud/common_resources.proto"),
220222
}
221-
got := gapicProtocArgs(apiProtos, additionalProtos, googleapisDir, "gapic-out", []string{"opt1", "opt2"})
223+
srcCfg := sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil)
224+
got := gapicProtocArgs(apiProtos, additionalProtos, srcCfg, "gapic-out", []string{"opt1", "opt2"})
222225
want := []string{
223226
"--experimental_allow_proto3_optional",
224227
"-I=" + googleapisDir,
@@ -336,19 +339,18 @@ func TestGenerateAPI(t *testing.T) {
336339
if err != nil {
337340
t.Fatal(err)
338341
}
339-
err = generateAPI(
340-
t.Context(),
341-
cfg,
342-
&config.API{Path: "google/cloud/secretmanager/v1"},
343-
library,
344-
googleapisDir,
345-
outdir,
346-
&repoMetadata{
342+
err = generateAPI(t.Context(), generateAPIParams{
343+
cfg: cfg,
344+
api: &config.API{Path: "google/cloud/secretmanager/v1"},
345+
library: library,
346+
srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil),
347+
outdir: outdir,
348+
metadata: &repoMetadata{
347349
NamePretty: "Secret Manager",
348350
APIDescription: "Secret Manager API",
349351
},
350-
apiCfg,
351-
)
352+
apiCfg: apiCfg,
353+
})
352354
if err != nil {
353355
t.Fatal(err)
354356
}
@@ -401,18 +403,17 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) {
401403
if err != nil {
402404
t.Fatal(err)
403405
}
404-
err = generateAPI(
405-
t.Context(),
406-
cfg,
407-
&config.API{Path: "google/cloud/gkehub/policycontroller/v1beta"},
408-
library,
409-
googleapisDir,
410-
outdir,
411-
&repoMetadata{
406+
err = generateAPI(t.Context(), generateAPIParams{
407+
cfg: cfg,
408+
api: &config.API{Path: "google/cloud/gkehub/policycontroller/v1beta"},
409+
library: library,
410+
srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil),
411+
outdir: outdir,
412+
metadata: &repoMetadata{
412413
NamePretty: "GKE Hub API",
413414
},
414-
apiCfg,
415-
)
415+
apiCfg: apiCfg,
416+
})
416417
if err != nil {
417418
t.Fatal(err)
418419
}
@@ -462,10 +463,18 @@ func TestGenerateAPI_NoTools(t *testing.T) {
462463
if err != nil {
463464
t.Fatal(err)
464465
}
465-
err = generateAPI(t.Context(), cfg, api, library, googleapisDir, outdir, &repoMetadata{
466-
NamePretty: "Secret Manager",
467-
APIDescription: "Secret Manager API",
468-
}, apiCfg)
466+
err = generateAPI(t.Context(), generateAPIParams{
467+
cfg: cfg,
468+
api: api,
469+
library: library,
470+
srcCfg: sources.NewSourceConfig(&sources.Sources{Googleapis: googleapisDir}, nil),
471+
outdir: outdir,
472+
metadata: &repoMetadata{
473+
NamePretty: "Secret Manager",
474+
APIDescription: "Secret Manager API",
475+
},
476+
apiCfg: apiCfg,
477+
})
469478
if err != nil {
470479
t.Fatal(err)
471480
}

‎internal/librarian/java/postprocess.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ type postProcessParams struct {
5050
metadata *repoMetadata
5151
outDir string
5252
apiBase string
53-
googleapisDir string
53+
protoSourceDir string
5454
apiProtos []string
5555
includeSamples bool
5656
}
@@ -322,7 +322,7 @@ func restructureModules(p postProcessParams, destRoot string) error {
322322
return err
323323
}
324324
// Copy proto files to proto-*/src/main/proto
325-
if err := copyProtos(p.googleapisDir, p.apiProtos, protoFilesDestDir); err != nil {
325+
if err := copyProtos(p.protoSourceDir, p.apiProtos, protoFilesDestDir); err != nil {
326326
return fmt.Errorf("failed to copy proto files: %w", err)
327327
}
328328
return nil
@@ -384,10 +384,10 @@ func deriveLastReleasedVersion(v string) (string, error) {
384384
return sv.String(), nil
385385
}
386386

387-
func copyProtos(googleapisDir string, protos []string, destDir string) error {
387+
func copyProtos(protoSourceDir string, protos []string, destDir string) error {
388388
for _, proto := range protos {
389-
// Calculate relative path from googleapisDir to preserve directory structure
390-
rel, err := filepath.Rel(googleapisDir, proto)
389+
// Calculate relative path from protoSourceDir to preserve directory structure
390+
rel, err := filepath.Rel(protoSourceDir, proto)
391391
if err != nil {
392392
return fmt.Errorf("failed to calculate relative path for %s: %w", proto, err)
393393
}

0 commit comments

Comments
 (0)