Skip to content

Commit 074059d

Browse files
yangyzsyangyzs
authored
feat(internal/librarian/java): integrate native Go postprocessor into Java generator (#6768)
Enables the Go-native postprocessor pipeline in the Java generator when the legacy owlbot.py script is absent. Added unit tests for the Go-native pipeline. For #6516
1 parent b2ff68c commit 074059d

3 files changed

Lines changed: 192 additions & 35 deletions

File tree

‎internal/librarian/java/generate_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,10 @@ func TestGenerateAPI(t *testing.T) {
271271
testhelper.RequireCommand(t, "protoc-gen-java_gapic")
272272
testhelper.RequireCommand(t, "protoc-gen-java_grpc")
273273
outdir := t.TempDir()
274+
// Force routing to the legacy owlbot.py postprocessor.
275+
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil {
276+
t.Fatal(err)
277+
}
274278
cfg := &config.Config{
275279
Repo: "googleapis/google-cloud-java",
276280
Default: &config.Default{
@@ -337,6 +341,10 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) {
337341
testhelper.RequireCommand(t, "protoc")
338342
testhelper.RequireCommand(t, "protoc-gen-java_grpc")
339343
outdir := t.TempDir()
344+
// Force routing to the legacy owlbot.py postprocessor.
345+
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil {
346+
t.Fatal(err)
347+
}
340348
cfg := &config.Config{
341349
Repo: "googleapis/google-cloud-java",
342350
Default: &config.Default{
@@ -474,6 +482,10 @@ func TestGenerateAPI_WithAdditionalProtosToGenerateAndCopy(t *testing.T) {
474482
testhelper.RequireCommand(t, "protoc-gen-java_gapic")
475483
testhelper.RequireCommand(t, "protoc-gen-java_grpc")
476484
outdir := t.TempDir()
485+
// Force routing to the legacy owlbot.py postprocessor.
486+
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil {
487+
t.Fatal(err)
488+
}
477489
cfg := &config.Config{
478490
Repo: "googleapis/google-cloud-java",
479491
Default: &config.Default{
@@ -1044,6 +1056,10 @@ func TestGenerateAPI_Gating(t *testing.T) {
10441056
} {
10451057
t.Run(test.name, func(t *testing.T) {
10461058
outdir := t.TempDir()
1059+
// Force routing to the legacy owlbot.py postprocessor.
1060+
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil {
1061+
t.Fatal(err)
1062+
}
10471063
api := &config.API{Path: "google/cloud/secretmanager/v1"}
10481064
cfg := &config.Config{
10491065
Repo: "googleapis/google-cloud-java",

‎internal/librarian/java/postprocess.go‎

Lines changed: 84 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/googleapis/librarian/internal/config"
3030
"github.com/googleapis/librarian/internal/filesystem"
3131
"github.com/googleapis/librarian/internal/license"
32+
"github.com/googleapis/librarian/internal/postprocessing"
3233
"github.com/googleapis/librarian/internal/serviceconfig"
3334
)
3435

@@ -70,18 +71,40 @@ type libraryPostProcessParams struct {
7071
// TODO(https://github.com/googleapis/librarian/issues/6627): Remove legacy owlbot.py
7172
// postprocessing execution once native Go postprocessing is enabled.
7273
func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error {
73-
if err := createOrVerifyOwlbotPy(params.outDir); err != nil {
74-
return err
75-
}
76-
bomVersion, err := findBOMVersion(params.cfg)
77-
if err != nil {
78-
return err
79-
}
80-
if err := removeKeptFilesFromStaging(params.library, params.outDir); err != nil {
81-
return fmt.Errorf("failed to remove kept files from staging: %w", err)
74+
owlbotPath := filepath.Join(params.outDir, "owlbot.py")
75+
_, err := os.Stat(owlbotPath)
76+
if err != nil && !errors.Is(err, fs.ErrNotExist) {
77+
return fmt.Errorf("failed to check for owlbot.py: %w", err)
8278
}
83-
if err := runOwlBot(ctx, params.library, params.outDir, bomVersion); err != nil {
84-
return fmt.Errorf("%w: %w", errRunOwlBot, err)
79+
owlbotExists := err == nil
80+
81+
if owlbotExists {
82+
if err := createOrVerifyOwlbotPy(params.outDir); err != nil {
83+
return err
84+
}
85+
bomVersion, err := findBOMVersion(params.cfg)
86+
if err != nil {
87+
return err
88+
}
89+
if err := removeKeptFilesFromStaging(params.library, params.outDir); err != nil {
90+
return fmt.Errorf("failed to remove kept files from staging: %w", err)
91+
}
92+
if err := runOwlBot(ctx, params.library, params.outDir, bomVersion); err != nil {
93+
return fmt.Errorf("%w: %w", errRunOwlBot, err)
94+
}
95+
} else {
96+
if params.library != nil && params.library.Postprocess != nil {
97+
if err := postprocessing.Apply(params.outDir, params.library.Postprocess); err != nil {
98+
return err
99+
}
100+
}
101+
var keepSet map[string]bool
102+
if params.library != nil {
103+
keepSet = toKeepSet(params.library.Keep)
104+
}
105+
if err := renderREADME(params, keepSet); err != nil {
106+
return fmt.Errorf("failed to render README: %w", err)
107+
}
85108
}
86109

87110
monorepoVersion, err := findMonorepoVersion(params.cfg)
@@ -140,23 +163,54 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error {
140163
if err := copyFiles(params); err != nil {
141164
return fmt.Errorf("failed to copy files: %w", err)
142165
}
143-
if err := restructureToStaging(params); err != nil {
144-
return fmt.Errorf("failed to restructure to staging: %w", err)
145-
}
146166

147-
// Generate clirr-ignored-differences.xml for the proto module.
148-
// We target the staging directory because runOwlBot hasn't moved the files
149-
// to their final destination yet.
150-
coords := params.coords()
151-
protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID)
152-
shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic)
153-
if err != nil {
154-
return fmt.Errorf("failed to check for clirr ignore file: %w", err)
167+
owlbotPath := filepath.Join(params.outDir, "owlbot.py")
168+
_, err := os.Stat(owlbotPath)
169+
if err != nil && !errors.Is(err, fs.ErrNotExist) {
170+
return fmt.Errorf("failed to check for owlbot.py: %w", err)
155171
}
156-
if shouldGenerate {
157-
protoModuleStagingRoot := filepath.Join(stagingDir(params.outDir), params.apiBase, coords.Proto.ArtifactID)
158-
if err := generateClirrIgnore(protoModuleStagingRoot); err != nil {
159-
return fmt.Errorf("failed to generate clirr ignore file: %w", err)
172+
owlbotExists := err == nil
173+
174+
if !owlbotExists {
175+
var keepSet map[string]bool
176+
if params.library != nil {
177+
keepSet = toKeepSet(params.library.Keep)
178+
}
179+
if err := restructureToLibrary(params, params.outDir, keepSet); err != nil {
180+
return fmt.Errorf("failed to restructure to library root: %w", err)
181+
}
182+
183+
coords := params.coords()
184+
// Generate clirr-ignored-differences.xml for the proto module.
185+
protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID)
186+
shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic)
187+
if err != nil {
188+
return fmt.Errorf("failed to check for clirr ignore file: %w", err)
189+
}
190+
if shouldGenerate {
191+
if err := generateClirrIgnore(protoModuleRepoRoot); err != nil {
192+
return fmt.Errorf("failed to generate clirr ignore file: %w", err)
193+
}
194+
}
195+
} else {
196+
if err := restructureToStaging(params); err != nil {
197+
return fmt.Errorf("failed to restructure to staging: %w", err)
198+
}
199+
200+
// Generate clirr-ignored-differences.xml for the proto module.
201+
// We target the staging directory because runOwlBot hasn't moved the files
202+
// to their final destination yet.
203+
coords := params.coords()
204+
protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID)
205+
shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic)
206+
if err != nil {
207+
return fmt.Errorf("failed to check for clirr ignore file: %w", err)
208+
}
209+
if shouldGenerate {
210+
protoModuleStagingRoot := filepath.Join(stagingDir(params.outDir), params.apiBase, coords.Proto.ArtifactID)
211+
if err := generateClirrIgnore(protoModuleStagingRoot); err != nil {
212+
return fmt.Errorf("failed to generate clirr ignore file: %w", err)
213+
}
160214
}
161215
}
162216

@@ -560,8 +614,8 @@ func ApplyMoveActionsToLibrary(actions []moveAction, destRoot string, keepSet ma
560614
return nil
561615
}
562616

563-
// ToKeepSet normalizes a list of keep paths into a lookup map.
564-
func ToKeepSet(keep []string) map[string]bool {
617+
// toKeepSet normalizes a list of keep paths into a lookup map.
618+
func toKeepSet(keep []string) map[string]bool {
565619
keepSet := make(map[string]bool, len(keep))
566620
for _, k := range keep {
567621
normalized := strings.TrimSuffix(filepath.ToSlash(k), "/")
@@ -570,9 +624,9 @@ func ToKeepSet(keep []string) map[string]bool {
570624
return keepSet
571625
}
572626

573-
// RestructureToLibrary moves all generated source code to the library root directories.
627+
// restructureToLibrary moves all generated source code to the library root directories.
574628
// It also removes conflicting files, and copies public proto files to the library.
575-
func RestructureToLibrary(params postProcessParams, destRoot string, keepSet map[string]bool) error {
629+
func restructureToLibrary(params postProcessParams, destRoot string, keepSet map[string]bool) error {
576630
tempProtoSrcDir := params.protoDir()
577631
isCommonProtos := params.library.Name == commonProtosLibrary
578632
if !isCommonProtos {

‎internal/librarian/java/postprocess_test.go‎

Lines changed: 92 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ import (
3636
func TestPostProcessAPI(t *testing.T) {
3737
t.Parallel()
3838
outdir := t.TempDir()
39+
// Force routing to the legacy owlbot.py postprocessor.
40+
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("# dummy"), 0755); err != nil {
41+
t.Fatal(err)
42+
}
3943
libraryName := "secretmanager"
4044
apiBase := "v1"
4145
gapicDir := filepath.Join(outdir, apiBase, "gapic")
@@ -1336,7 +1340,7 @@ func TestRestructureToLibrary(t *testing.T) {
13361340
includeSamples: tc.includeSamples,
13371341
apiBase: "v1",
13381342
}
1339-
if err := RestructureToLibrary(params, destDir, nil); err != nil {
1343+
if err := restructureToLibrary(params, destDir, nil); err != nil {
13401344
t.Fatal(err)
13411345
}
13421346
gotFiles := readDirFiles(t, destDir)
@@ -1347,7 +1351,7 @@ func TestRestructureToLibrary(t *testing.T) {
13471351
}
13481352
}
13491353

1350-
// TestRestructureToLibrary_OverwritesExistingFiles verifies that existing files in the destination are overwritten.
1354+
// TestRestructureToLibrary_OverwritesExistingFiles verifies that restructureToLibrary overwrites existing files in the destination.
13511355
func TestRestructureToLibrary_OverwritesExistingFiles(t *testing.T) {
13521356
t.Parallel()
13531357
srcDir := t.TempDir()
@@ -1372,7 +1376,7 @@ func TestRestructureToLibrary_OverwritesExistingFiles(t *testing.T) {
13721376
apiBase: "v1",
13731377
}
13741378
// Pass nil keepSet to expect default overwriting of conflicting files.
1375-
if err := RestructureToLibrary(params, destDir, nil); err != nil {
1379+
if err := restructureToLibrary(params, destDir, nil); err != nil {
13761380
t.Fatal(err)
13771381
}
13781382
gotFiles := readDirFiles(t, destDir)
@@ -1404,7 +1408,7 @@ func TestRestructureToLibrary_CommonProtos(t *testing.T) {
14041408
includeSamples: false,
14051409
apiBase: "v1",
14061410
}
1407-
if err := RestructureToLibrary(params, destDir, nil); err != nil {
1411+
if err := restructureToLibrary(params, destDir, nil); err != nil {
14081412
t.Fatal(err)
14091413
}
14101414
gotFiles := readDirFiles(t, destDir)
@@ -1416,6 +1420,89 @@ func TestRestructureToLibrary_CommonProtos(t *testing.T) {
14161420
}
14171421
}
14181422

1423+
// TestPostProcessAPI_Go verifies that Go-native postprocessor correctly restructures
1424+
// generated Java files to their target directories and cleans up intermediate files.
1425+
func TestPostProcessAPI_Go(t *testing.T) {
1426+
t.Parallel()
1427+
dir := t.TempDir()
1428+
writeFiles(t, dir, map[string]string{
1429+
"v1/gapic/src/main/java/Foo.java": "class Foo {}",
1430+
"v1/grpc/dummy": "",
1431+
"v1/proto/dummy": "",
1432+
})
1433+
library := &config.Library{
1434+
Name: "test-lib",
1435+
APIs: []*config.API{{Path: "google/cloud/test/v1", Java: &config.JavaAPI{}}},
1436+
Java: &config.JavaModule{GroupID: "com.google.cloud", ArtifactID: "google-cloud-test", ReleasedVersion: "1.2.3"},
1437+
}
1438+
postParams := postProcessParams{
1439+
cfg: &config.Config{},
1440+
library: library,
1441+
javaAPI: library.APIs[0].Java,
1442+
outDir: dir,
1443+
includeSamples: true,
1444+
apiBase: "v1",
1445+
}
1446+
if err := postProcessAPI(t.Context(), postParams); err != nil {
1447+
t.Fatal(err)
1448+
}
1449+
// Verify that files are relocated directly to target paths and staging is skipped.
1450+
want := map[string]string{
1451+
"google-cloud-test/src/main/java/Foo.java": "class Foo {}",
1452+
"proto-google-cloud-test-v1/src/main/java/dummy": "",
1453+
"grpc-google-cloud-test-v1/src/main/java/dummy": "",
1454+
}
1455+
got := readDirFiles(t, dir)
1456+
if diff := cmp.Diff(want, got); diff != "" {
1457+
t.Errorf("mismatch (-want +got):\n%s", diff)
1458+
}
1459+
}
1460+
1461+
// TestPostProcessLibrary_Go verifies that library-level postprocessing tasks
1462+
// (such as text replacements, POM updates, and README generation) execute
1463+
// correctly in the Go-native flow.
1464+
func TestPostProcessLibrary_Go(t *testing.T) {
1465+
t.Parallel()
1466+
dir := t.TempDir()
1467+
writeFiles(t, dir, map[string]string{"google-cloud-test/src/main/java/Foo.java": "class Foo {}"})
1468+
library := &config.Library{
1469+
Name: "test-lib",
1470+
Java: &config.JavaModule{
1471+
GroupID: "com.google.cloud",
1472+
ArtifactID: "google-cloud-test",
1473+
// Disable syncPOMs to simplify config requirements.
1474+
SkipPOMUpdates: true,
1475+
},
1476+
// Disable renderREADME to simplify config requirements.
1477+
Keep: []string{"README.md"},
1478+
Postprocess: &config.Postprocess{
1479+
Replace: []config.ReplaceConfig{
1480+
{Path: "google-cloud-test/src/main/java/Foo.java", Original: "class Foo", Replacement: "class RenamedFoo"},
1481+
},
1482+
},
1483+
}
1484+
params := libraryPostProcessParams{
1485+
cfg: &config.Config{
1486+
Libraries: []*config.Library{
1487+
{Name: "google-cloud-java", Version: "1.2.3"},
1488+
{Name: "google-cloud-pom-parent", Version: "1.2.3"},
1489+
},
1490+
},
1491+
library: library,
1492+
outDir: dir,
1493+
metadata: &repoMetadata{},
1494+
}
1495+
if err := postProcessLibrary(t.Context(), params); err != nil {
1496+
t.Fatal(err)
1497+
}
1498+
// Verify postprocessing rules were applied.
1499+
want := map[string]string{"google-cloud-test/src/main/java/Foo.java": "class RenamedFoo {}"}
1500+
got := readDirFiles(t, dir)
1501+
if diff := cmp.Diff(want, got); diff != "" {
1502+
t.Errorf("mismatch (-want +got):\n%s", diff)
1503+
}
1504+
}
1505+
14191506
func writeFiles(t *testing.T, dir string, files map[string]string) {
14201507
t.Helper()
14211508
if err := os.MkdirAll(dir, 0755); err != nil {
@@ -1462,7 +1549,7 @@ func readDirFiles(t *testing.T, dir string) map[string]string {
14621549
func TestToKeepSet(t *testing.T) {
14631550
t.Parallel()
14641551
input := []string{"foo/", "bar/baz", "qux/", ""}
1465-
got := ToKeepSet(input)
1552+
got := toKeepSet(input)
14661553
want := map[string]bool{
14671554
"foo": true,
14681555
"bar/baz": true,

0 commit comments

Comments
 (0)