feat(internal/librarian): allow language-specific parallelism (#4005)

Refactors generation such that each phase of clean/generate/format is
treated as a whole across all libraries, moving towards
language-specific determination of when to perform operations in
parallel within each phase.

This commit only delegates that choice for generation, but the same
pattern can be followed for cleaning and formatting when there's a
desire to do so.

Fixes #3933

Author: jskeet

internal/librarian/dart/generate.go

@@ -26,10 +26,23 @@ import (
 	sidekickdart "github.com/googleapis/librarian/internal/sidekick/dart"
 	"github.com/googleapis/librarian/internal/sidekick/parser"
 	"github.com/googleapis/librarian/internal/sidekick/source"
+	"golang.org/x/sync/errgroup"
 )
 
-// Generate generates a Dart client library.
-func Generate(ctx context.Context, library *config.Library, sources *source.Sources) error {
+// GenerateLibraries generates all the given libraries in parallel.
+func GenerateLibraries(ctx context.Context, libraries []*config.Library, sources *source.Sources) error {
+	// Generate all libraries in parallel.
+	g, gctx := errgroup.WithContext(ctx)
+	for _, lib := range libraries {
+		g.Go(func() error {
+			return generate(gctx, lib, sources)
+		})
+	}
+	return g.Wait()
+}
+
+// generate generates a Dart client library.
+func generate(ctx context.Context, library *config.Library, sources *source.Sources) error {
 	modelConfig, err := toModelConfig(library, library.APIs[0], sources)
 	if err != nil {
 		return err

internal/librarian/dart/generate_test.go

@@ -15,6 +15,7 @@
 package dart
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
@@ -26,6 +27,89 @@ import (
 	"github.com/googleapis/librarian/internal/testhelper"
 )
 
+// TestGenerateLibraries performs simple testing that multiple libraries can
+// be generated. Only the presence of a single expected file per library is
+// performed; TestGenerate is responsible for more detailed testing of
+// per-library generation.
+func TestGenerateLibraries(t *testing.T) {
+	testhelper.RequireCommand(t, "protoc")
+	testhelper.RequireCommand(t, "dart")
+
+	googleapisDir, err := filepath.Abs("../../testdata/googleapis")
+	if err != nil {
+		t.Fatal(err)
+	}
+	outDir := t.TempDir()
+	libraries := []*config.Library{
+		{
+			Name:          "google_cloud_rpc",
+			APIs:          []*config.API{{Path: "google/rpc"}},
+			CopyrightYear: "2025",
+			Dart: &config.DartPackage{
+				IssueTrackerURL: "https://placeholder",
+				Packages: map[string]string{
+					"package:google_cloud_protobuf": "^0.5.0",
+				},
+			},
+		},
+		{
+			Name:          "google_type",
+			APIs:          []*config.API{{Path: "google/type"}},
+			CopyrightYear: "2025",
+			Dart: &config.DartPackage{
+				IssueTrackerURL: "https://placeholder",
+				Packages: map[string]string{
+					"package:google_cloud_protobuf": "^0.5.0",
+				},
+			},
+		},
+	}
+	for _, library := range libraries {
+		library.Output = filepath.Join(outDir, "generated", library.Name)
+	}
+	sources := &source.Sources{
+		Googleapis: googleapisDir,
+	}
+	if err := GenerateLibraries(t.Context(), libraries, sources); err != nil {
+		t.Fatal(err)
+	}
+	// Just check that a pubspec.yaml has been created for each library.
+	for _, library := range libraries {
+		expectedPubspec := filepath.Join(library.Output, "pubspec.yaml")
+		_, err := os.Stat(expectedPubspec)
+		if err != nil {
+			t.Errorf("Stat(%s) returned error: %v", expectedPubspec, err)
+		}
+	}
+}
+
+func TestGenerateLibraries_Error(t *testing.T) {
+	testhelper.RequireCommand(t, "protoc")
+	testhelper.RequireCommand(t, "dart")
+
+	googleapisDir, err := filepath.Abs("../../testdata/googleapis")
+	if err != nil {
+		t.Fatal(err)
+	}
+	outDir := t.TempDir()
+	libraries := []*config.Library{
+		{
+			Name:                "broken",
+			SpecificationFormat: "invalid",
+			Output:              filepath.Join(outDir, "broken"),
+			APIs:                []*config.API{{Path: "broken"}},
+		},
+	}
+	sources := &source.Sources{
+		Googleapis: googleapisDir,
+	}
+	gotErr := GenerateLibraries(t.Context(), libraries, sources)
+	wantErr := errInvalidSpecificationFormat
+	if !errors.Is(gotErr, wantErr) {
+		t.Errorf("GenerateLibraries error = %v, wantErr %v", gotErr, wantErr)
+	}
+}
+
 func TestGenerate(t *testing.T) {
 	testhelper.RequireCommand(t, "protoc")
 	testhelper.RequireCommand(t, "dart")
@@ -66,7 +150,7 @@ func TestGenerate(t *testing.T) {
 	sources := &source.Sources{
 		Googleapis: googleapisDir,
 	}
-	if err := Generate(t.Context(), library, sources); err != nil {
+	if err := generate(t.Context(), library, sources); err != nil {
 		t.Fatal(err)
 	}
 	if err := Format(t.Context(), library); err != nil {

internal/librarian/fake.go

@@ -30,6 +30,15 @@ func fakeBumpLibrary(lib *config.Library, nextVersion string) error {
 	return nil
 }
 
+func fakeGenerateLibraries(libraries []*config.Library) error {
+	for _, library := range libraries {
+		if err := fakeGenerate(library); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func fakeGenerate(library *config.Library) error {
 	if _, err := os.Stat(library.Output); err != nil {
 		if !os.IsNotExist(err) {

internal/librarian/fake_test.go

@@ -23,7 +23,7 @@ import (
 	"github.com/googleapis/librarian/internal/config"
 )
 
-func TestGenerate(t *testing.T) {
+func TestGenerateLibraries(t *testing.T) {
 	const (
 		libraryName = "test-library"
 		outputDir   = "output"
@@ -35,7 +35,7 @@ func TestGenerate(t *testing.T) {
 
 	tmpDir := t.TempDir()
 	t.Chdir(tmpDir)
-	if err := generate(t.Context(), "fake", library, "", nil); err != nil {
+	if err := generateLibraries(t.Context(), "fake", []*config.Library{library}, "", nil); err != nil {
 		t.Fatal(err)
 	}
 

internal/librarian/generate.go

@@ -29,7 +29,6 @@ import (
 	"github.com/googleapis/librarian/internal/sidekick/source"
 	"github.com/googleapis/librarian/internal/yaml"
 	"github.com/urfave/cli/v3"
-	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -76,23 +75,20 @@ func runGenerate(ctx context.Context, cfg *config.Config, all bool, libraryName
 	if cfg.Sources == nil {
 		return errEmptySources
 	}
-	return generateLibraries(ctx, all, cfg, libraryName)
-}
 
-func generateLibraries(ctx context.Context, all bool, cfg *config.Config, libraryName string) error {
 	googleapisDir, rustDartSources, err := LoadSources(ctx, cfg)
 	if err != nil {
 		return err
 	}
 
-	// Prepare and clean libraries sequentially.
-	// This avoids race conditions when output directories are nested.
+	// Prepare the libraries to generate by skipping as specified and applying
+	// defaults.
 	var libraries []*config.Library
 	for _, lib := range cfg.Libraries {
 		if !shouldGenerate(lib, all, libraryName) {
 			continue
 		}
-		prepared, err := prepareLibrary(cfg.Language, lib, cfg.Default)
+		prepared, err := applyDefaults(cfg.Language, lib, cfg.Default)
 		if err != nil {
 			return err
 		}
@@ -110,22 +106,17 @@ func generateLibraries(ctx context.Context, all bool, cfg *config.Config, librar
 		return fmt.Errorf("%w: %q", ErrLibraryNotFound, libraryName)
 	}
 
-	// Generate all libraries in parallel.
-	g, gctx := errgroup.WithContext(ctx)
-	for _, lib := range libraries {
-		g.Go(func() error {
-			return generate(gctx, cfg.Language, lib, googleapisDir, rustDartSources)
-		})
+	// Clean, generate and format libraries. Each of these steps is completed
+	// before the next one starts, but each language can choose whether to
+	// implement the step in parallel across all libraries or in sequence.
+	if err := cleanLibraries(cfg.Language, libraries); err != nil {
+		return err
 	}
-	if err := g.Wait(); err != nil {
+	if err := generateLibraries(ctx, cfg.Language, libraries, googleapisDir, rustDartSources); err != nil {
 		return err
 	}
-
-	// Format all libraries sequentially.
-	for _, lib := range libraries {
-		if err := formatLibrary(ctx, cfg.Language, lib); err != nil {
-			return err
-		}
+	if err := formatLibraries(ctx, cfg.Language, libraries); err != nil {
+		return err
 	}
 	return postGenerate(ctx, cfg.Language)
 }
@@ -158,6 +149,85 @@ func LoadSources(ctx context.Context, cfg *config.Config) (string, *source.Sourc
 	return googleapisDir, rustDartSources, nil
 }
 
+// cleanLibraries iterates over all the given libraries sequentially,
+// delegating to language-specific code to clean each library.
+func cleanLibraries(language string, libraries []*config.Library) error {
+	for _, library := range libraries {
+		switch language {
+		case languageFake:
+			// No cleaning needed.
+		case languageDart, languagePython:
+			if err := checkAndClean(library.Output, library.Keep); err != nil {
+				return err
+			}
+		case languageGo:
+			if err := golang.Clean(library); err != nil {
+				return err
+			}
+		case languageRust:
+			keep, err := rust.Keep(library)
+			if err != nil {
+				return fmt.Errorf("library %q: %w", library.Name, err)
+			}
+			if err := checkAndClean(library.Output, keep); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// generateLibraries delegates to language-specific code to generate all the
+// given libraries.
+func generateLibraries(ctx context.Context, language string, libraries []*config.Library, googleapisDir string, src *source.Sources) error {
+	switch language {
+	case languageFake:
+		return fakeGenerateLibraries(libraries)
+	case languageDart:
+		return dart.GenerateLibraries(ctx, libraries, src)
+	case languagePython:
+		return python.GenerateLibraries(ctx, libraries, googleapisDir)
+	case languageGo:
+		return golang.GenerateLibraries(ctx, libraries, googleapisDir)
+	case languageRust:
+		return rust.GenerateLibraries(ctx, libraries, src)
+	default:
+		return fmt.Errorf("language %q does not support generation", language)
+	}
+}
+
+// formatLibraries iterates over all the given libraries sequentially,
+// delegating to language-specific code to format each library.
+func formatLibraries(ctx context.Context, language string, libraries []*config.Library) error {
+	for _, library := range libraries {
+		switch language {
+		case languageFake:
+			if err := fakeFormat(library); err != nil {
+				return err
+			}
+		case languageDart:
+			if err := dart.Format(ctx, library); err != nil {
+				return err
+			}
+		case languageGo:
+			if err := golang.Format(ctx, library); err != nil {
+				return err
+			}
+		case languageRust:
+			if err := rust.Format(ctx, library); err != nil {
+				return err
+			}
+		case languagePython:
+			// TODO(https://github.com/googleapis/librarian/issues/3730): separate
+			// generation and formatting for Python.
+			return nil
+		default:
+			return fmt.Errorf("language %q does not support formatting", language)
+		}
+	}
+	return nil
+}
+
 // postGenerate performs repository-level actions after all individual
 // libraries have been generated.
 func postGenerate(ctx context.Context, language string) error {
@@ -201,76 +271,3 @@ func shouldGenerate(lib *config.Library, all bool, libraryName string) bool {
 	}
 	return all || lib.Name == libraryName
 }
-
-// prepareLibrary applies defaults and cleans the output directory.
-func prepareLibrary(language string, lib *config.Library, defaults *config.Default) (*config.Library, error) {
-	library, err := applyDefaults(language, lib, defaults)
-	if err != nil {
-		return nil, err
-	}
-	switch language {
-	case languageFake:
-		// No cleaning needed.
-	case languageDart, languagePython:
-		if err := checkAndClean(library.Output, library.Keep); err != nil {
-			return nil, err
-		}
-	case languageGo:
-		return golang.Clean(library)
-	case languageRust:
-		keep, err := rust.Keep(library)
-		if err != nil {
-			return nil, fmt.Errorf("library %q: %w", library.Name, err)
-		}
-		if err := checkAndClean(library.Output, keep); err != nil {
-			return nil, err
-		}
-	}
-	return library, nil
-}
-
-func generate(ctx context.Context, language string, library *config.Library, googleapisDir string, src *source.Sources) error {
-	switch language {
-	case languageFake:
-		if err := fakeGenerate(library); err != nil {
-			return err
-		}
-	case languageDart:
-		if err := dart.Generate(ctx, library, src); err != nil {
-			return err
-		}
-	case languagePython:
-		if err := python.Generate(ctx, library, googleapisDir); err != nil {
-			return err
-		}
-	case languageGo:
-		if err := golang.Generate(ctx, library, googleapisDir); err != nil {
-			return err
-		}
-	case languageRust:
-		if err := rust.Generate(ctx, library, src); err != nil {
-			return err
-		}
-	default:
-		return fmt.Errorf("language %q does not support generation", language)
-	}
-	return nil
-}
-
-func formatLibrary(ctx context.Context, language string, library *config.Library) error {
-	switch language {
-	case languageFake:
-		return fakeFormat(library)
-	case languageDart:
-		return dart.Format(ctx, library)
-	case languageGo:
-		return golang.Format(ctx, library)
-	case languageRust:
-		return rust.Format(ctx, library)
-	case languagePython:
-		// TODO(https://github.com/googleapis/librarian/issues/3730): separate
-		// generation and formatting for Python.
-		return nil
-	}
-	return fmt.Errorf("language %q does not support formatting", language)
-}