feat: generate extension catalogs

Signed-off-by: Niccolò Fei <niccolo.fei@enterprisedb.com>

Author: NiccoloFei

.github/workflows/bake.yml

@@ -89,3 +89,16 @@ jobs:
       extension_name: ${{ matrix.extension }}
     secrets:
       SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+
+  Catalogs:
+    name: Update Catalogs
+    needs: Bake
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: write
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - name: Repository Dispatch
+        uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4
+        with:
+          event-type: update-catalogs

.github/workflows/update-catalogs.yml

@@ -0,0 +1,57 @@
+name: Update Catalogs
+
+on:
+  workflow_dispatch:
+  repository_dispatch:
+    types: [update-catalogs]
+
+permissions: read-all
+
+defaults:
+  run:
+    shell: "bash -Eeuo pipefail -x {0}"
+
+jobs:
+  update-catalogs:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        with:
+          persist-credentials: false
+
+      - name: Checkout artifacts
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+        with:
+          path: artifacts
+          repository: cloudnative-pg/artifacts
+          token: ${{ secrets.REPO_GHA_PAT }}
+          ref: main
+
+      - name: Update catalogs
+        id: update-extension-catalogs
+        uses: dagger/dagger-for-github@d913e70051faf3b907d4dd96ef1161083c88c644 # v8.2.0
+        env:
+          # renovate: datasource=github-tags depName=dagger/dagger versioning=semver
+          DAGGER_VERSION: 0.19.10
+        with:
+          version: ${{ env.DAGGER_VERSION }}
+          verb: call
+          module: ./dagger/maintenance/
+          args: generate-catalogs --catalogs-dir artifacts/image-catalogs/ export --path artifacts/image-catalogs/
+
+      - name: Diff
+        working-directory: artifacts
+        run: |
+          git add -A .
+          git status
+          git diff --staged
+
+      - uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 # v9
+        if: github.ref == 'refs/heads/main'
+        with:
+          cwd: 'artifacts'
+          add: 'image-catalogs'
+          author_name: CloudNativePG Automated Updates
+          author_email: noreply@cnpg.com
+          message: 'chore: update extensions imageCatalogs'

dagger/maintenance/catalogs.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"path/filepath"
+	"slices"
+
+	"go.yaml.in/yaml/v3"
+
+	"dagger/maintenance/internal/dagger"
+)
+
+const (
+	LabelImageOS   = "images.cnpg.io/os"
+	LabelImageType = "images.cnpg.io/type"
+)
+
+type ImageVolumeSource struct {
+	Reference  string `yaml:"reference"`
+	PullPolicy string `yaml:"pullPolicy,omitempty"`
+}
+
+type ExtensionConfiguration struct {
+	Name                 string            `yaml:"name"`
+	ImageVolumeSource    ImageVolumeSource `yaml:"image"`
+	ExtensionControlPath []string          `yaml:"extensionControlPath,omitempty"`
+	DynamicLibraryPath   []string          `yaml:"dynamicLibraryPath,omitempty"`
+	LdLibraryPath        []string          `yaml:"ldLibraryPath,omitempty"`
+}
+
+type ImageCatalog struct {
+	APIVersion string `yaml:"apiVersion"`
+	Kind       string `yaml:"kind"`
+	Metadata   struct {
+		Name   string            `yaml:"name"`
+		Labels map[string]string `yaml:"labels"`
+	} `yaml:"metadata"`
+	Spec struct {
+		Images []struct {
+			Major      int                      `yaml:"major"`
+			Image      string                   `yaml:"image"`
+			Extensions []ExtensionConfiguration `yaml:"extensions,omitempty"`
+		} `yaml:"images"`
+	} `yaml:"spec"`
+}
+
+func getMinimalCatalogs(ctx context.Context, catalogsDir *dagger.Directory) ([]*ImageCatalog, error) {
+	entries, err := catalogsDir.Entries(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	var catalogs []*ImageCatalog
+
+	for _, entry := range entries {
+		if ext := filepath.Ext(entry); ext != ".yaml" && ext != ".yml" {
+			continue
+		}
+
+		content, err := catalogsDir.File(entry).Contents(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("while retrieving %s: %w", entry, err)
+		}
+
+		var catalog ImageCatalog
+		if err := yaml.Unmarshal([]byte(content), &catalog); err != nil {
+			return nil, fmt.Errorf("while decoding %s: %w", entry, err)
+		}
+
+		// Only keep ClusterImageCatalogs
+		if catalog.Kind != "ClusterImageCatalog" {
+			continue
+		}
+
+		// Only keep catalogs with minimal images
+		if catalog.Metadata.Labels[LabelImageType] != "minimal" {
+			continue
+		}
+
+		// Only keep catalogs for Supported Distros
+		catalogOS, ok := catalog.Metadata.Labels[LabelImageOS]
+		if !ok {
+			return nil, fmt.Errorf("while retrieving OS for %q catalog", entry)
+		}
+		if !slices.Contains(SupportedDistributions, catalogOS) {
+			continue
+		}
+
+		catalogs = append(catalogs, &catalog)
+	}
+
+	return catalogs, nil
+}
+
+func writeCatalogToDir(catalog *ImageCatalog, outDir *dagger.Directory) (*dagger.Directory, error) {
+	var buf bytes.Buffer
+	enc := yaml.NewEncoder(&buf)
+	enc.SetIndent(2)
+
+	if err := enc.Encode(catalog); err != nil {
+		return nil, fmt.Errorf("while encoding catalog %s: %w", catalog.Metadata.Name, err)
+	}
+	if err := enc.Close(); err != nil {
+		return nil, err
+	}
+
+	outName := fmt.Sprintf("catalog-extensions-%s.yaml", catalog.Metadata.Labels[LabelImageOS])
+
+	return outDir.WithNewFile(outName, buf.String()), nil
+}

dagger/maintenance/image.go

@@ -17,6 +17,11 @@ const (
 	DefaultDistribution = "trixie"
 )
 
+var SupportedDistributions = []string{
+	"bookworm",
+	"trixie",
+}
+
 // getImageAnnotations returns the OCI annotations given an image ref.
 func getImageAnnotations(imageRef string) (map[string]string, error) {
 	// Setting Insecure option to allow fetching images from local registries with no TLS
@@ -51,10 +56,16 @@ func getImageAnnotations(imageRef string) (map[string]string, error) {
 // getDefaultExtensionImage returns the default extension image for a given extension,
 // resolved from the metadata.
 func getDefaultExtensionImage(metadata *extensionMetadata) (string, error) {
-	packageVersion := metadata.Versions[DefaultDistribution][strconv.Itoa(DefaultPgMajor)]
+	return getExtensionImage(metadata, DefaultDistribution, DefaultPgMajor)
+}
+
+// getExtensionImage returns the extension image for a given distribution and pgMajor,
+// resolved from the metadata.
+func getExtensionImage(metadata *extensionMetadata, distribution string, pgMajor int) (string, error) {
+	packageVersion := metadata.Versions[distribution][strconv.Itoa(pgMajor)]
 	if packageVersion == "" {
 		return "", fmt.Errorf("no package version found for distribution %q and version %d",
-			DefaultDistribution, DefaultPgMajor)
+			distribution, pgMajor)
 	}
 
 	re := regexp.MustCompile(`^(\d+(?:\.\d+)+)`)
@@ -64,7 +75,7 @@ func getDefaultExtensionImage(metadata *extensionMetadata) (string, error) {
 	}
 	version := matches[1]
 	image := fmt.Sprintf("ghcr.io/cloudnative-pg/%s:%s-%d-%s",
-		metadata.ImageName, version, DefaultPgMajor, DefaultDistribution)
+		metadata.ImageName, version, pgMajor, distribution)
 
 	return image, nil
 }

dagger/maintenance/main.go

@@ -12,6 +12,8 @@ import (
 	"path"
 	"regexp"
 	"slices"
+	"sort"
+	"strconv"
 	"strings"
 	"text/template"
 	"time"
@@ -410,3 +412,94 @@ func (m *Maintenance) Test(
 
 	return nil
 }
+
+// Generate extension's ClusterImageCatalogs starting from a base set of catalogs
+func (m *Maintenance) GenerateCatalogs(
+	ctx context.Context,
+	// The source directory containing the extension folders. Defaults to the current directory
+	// +ignore=["dagger", ".github"]
+	// +defaultPath="/"
+	source *dagger.Directory,
+	// The directory containing the starting catalogs. Defaults to "/image-catalogs"
+	// +defaultPath="/image-catalogs"
+	catalogsDir *dagger.Directory,
+) (*dagger.Directory, error) {
+	outDir := dag.Directory()
+
+	catalogs, err := getMinimalCatalogs(ctx, catalogsDir)
+	if err != nil {
+		return nil, fmt.Errorf("while retrieving base catalogs: %w", err)
+	}
+
+	targetExtensions, err := getExtensions(ctx, source)
+	if err != nil {
+		return nil, fmt.Errorf("while retrieving extensions: %w", err)
+	}
+	if len(targetExtensions) == 0 {
+		return nil, fmt.Errorf("no extensions found in source directory")
+	}
+
+	catalogWritten := false
+	for _, catalog := range catalogs {
+		catalogOS, ok := catalog.Metadata.Labels[LabelImageOS]
+		if !ok {
+			return nil, fmt.Errorf("while retrieving OS for %q catalog", catalog.Metadata.Name)
+		}
+
+		for dir, extension := range targetExtensions {
+			matrix, err := parseBuildMatrix(ctx, source, dir)
+			if err != nil {
+				return nil, fmt.Errorf("while parsing build Matrix for extension %s: %w", extension, err)
+			}
+			if !slices.Contains(matrix.Distributions, catalogOS) {
+				continue
+			}
+
+			for i := range catalog.Spec.Images {
+				img := &catalog.Spec.Images[i]
+				if !slices.Contains(matrix.MajorVersions, strconv.Itoa(img.Major)) {
+					continue
+				}
+
+				metadata, err := parseExtensionMetadata(ctx, source.Directory(dir))
+				if err != nil {
+					return nil, fmt.Errorf("while parsing extension %s metadata: %w", extension, err)
+				}
+
+				targetExtensionImage, err := getExtensionImage(metadata, catalogOS, img.Major)
+				if err != nil {
+					return nil, fmt.Errorf("while retrieving extension %s image: %w", extension, err)
+				}
+
+				extensionsConfig := ExtensionConfiguration{
+					Name: metadata.Name,
+					ImageVolumeSource: ImageVolumeSource{
+						Reference: targetExtensionImage,
+					},
+					ExtensionControlPath: metadata.ExtensionControlPath,
+					DynamicLibraryPath:   metadata.DynamicLibraryPath,
+					LdLibraryPath:        metadata.LdLibraryPath,
+				}
+
+				img.Extensions = append(img.Extensions, extensionsConfig)
+
+				// Sort extensions by name
+				sort.Slice(img.Extensions, func(i, j int) bool {
+					return img.Extensions[i].Name < img.Extensions[j].Name
+				})
+			}
+		}
+
+		outDir, err = writeCatalogToDir(catalog, outDir)
+		if err != nil {
+			return nil, fmt.Errorf("while writing catalog %s: %w", catalog.Metadata.Name, err)
+		}
+		catalogWritten = true
+	}
+
+	if !catalogWritten {
+		return nil, fmt.Errorf("no catalogs matched the selection criteria")
+	}
+
+	return outDir, nil
+}