fix: generate macros for all subpackages

toolkit/tools/internal/rpm/rpm.go

@@ -64,6 +64,51 @@ const (
 	MaxCPUDefine = "_smp_ncpus_max"
 )
 
+var (
+	// RpmSpecBuiltRPMRegex extracts the package name, optional epoch, version, release, distribution,
+	// and architecture from values returned by 'rpmspec --builtrpms' (also produced by
+	// QuerySPECForBuiltRPMs), which follow the `%{nevra}` format.
+	//
+	// Examples:
+	//
+	//	kernel-6.6.134.1-2.azl3.x86_64               -> Name: kernel,          Epoch: "", Version: 6.6.134.1, Release: 2,  Distribution: azl3, Architecture: x86_64
+	//	python3-perf-5.15.63.1-1.azl3.x86_64         -> Name: python3-perf,    Epoch: "", Version: 5.15.63.1, Release: 1,  Distribution: azl3, Architecture: x86_64
+	//	ca-certificates-1:3.0.0-14.azl3.noarch       -> Name: ca-certificates, Epoch: 1,  Version: 3.0.0,     Release: 14, Distribution: azl3, Architecture: noarch
+	//
+	// NOTE: regular expression based on the following assumptions:
+	//   - Package version and release values are not allowed to contain a hyphen character.
+	//   - Our tooling prevents the 'Release' tag from having any other form than '[[:digit:]]+%{?dist}'.
+	//   - The distribution tag is not allowed to contain a period or a hyphen.
+	//   - The architecture is not allowed to contain a period or a hyphen.
+	//   - When the package has a non-zero epoch, it appears immediately before the version as `epoch:`;
+	//     a missing epoch (the default `0`) is omitted entirely from the input.
+	//
+	// Regex breakdown:
+	//
+	//	^(.*)        <-- [index 1] package name (may contain hyphens)
+	//	-            <-- third-to-last hyphen separating the package name from its version
+	//	(?:(\d+):)?  <-- [index 2] optional epoch (digits) followed by a colon
+	//	([^-]+)      <-- [index 3] package version
+	//	-            <-- second-to-last hyphen separating the version from the release
+	//	([^-]+)      <-- [index 4] package release
+	//	\.           <-- second-to-last period separating the release from the distribution tag
+	//	([^.]+)      <-- [index 5] the distribution tag
+	//	\.           <-- last period separating the distribution tag from the architecture string
+	//	([^.]+)$     <-- [index 6] the architecture string
+	RpmSpecBuiltRPMRegex = regexp.MustCompile(`^(.*)-(?:(\d+):)?([^-]+)-([^-]+)\.([^.]+)\.([^.]+)$`)
+)
+
+// Index constants for capture groups of RpmSpecBuiltRPMRegex.
+const (
+	RpmSpecBuiltRPMRegexNameIndex = iota + 1
+	RpmSpecBuiltRPMRegexEpochIndex
+	RpmSpecBuiltRPMRegexVersionIndex
+	RpmSpecBuiltRPMRegexReleaseIndex
+	RpmSpecBuiltRPMRegexDistributionIndex
+	RpmSpecBuiltRPMRegexArchitectureIndex
+	RpmSpecBuiltRPMRegexMatchesCount
+)
+
 const (
 	packageFQNRegexMatchSubString  = iota
 	packageFQNRegexNameIndex       = iota

toolkit/tools/internal/rpm/rpm_test.go

@@ -725,3 +725,48 @@ func TestStripEpochFromPackageFullQualifiedNameWithInvalidInput(t *testing.T) {
 		})
 	}
 }
+
+func TestRpmSpecBuiltRPMRegexBasic(t *testing.T) {
+	input := "pkg-1.2.3-1.azl3.x86_64"
+	expectedResults := []string{"pkg", "", "1.2.3", "1", "azl3", "x86_64"}
+
+	matches := RpmSpecBuiltRPMRegex.FindStringSubmatch(input)
+	assert.Equal(t, RpmSpecBuiltRPMRegexMatchesCount, len(matches))
+
+	assert.Equal(t, expectedResults[0], matches[RpmSpecBuiltRPMRegexNameIndex])
+	assert.Equal(t, expectedResults[1], matches[RpmSpecBuiltRPMRegexEpochIndex])
+	assert.Equal(t, expectedResults[2], matches[RpmSpecBuiltRPMRegexVersionIndex])
+	assert.Equal(t, expectedResults[3], matches[RpmSpecBuiltRPMRegexReleaseIndex])
+	assert.Equal(t, expectedResults[4], matches[RpmSpecBuiltRPMRegexDistributionIndex])
+	assert.Equal(t, expectedResults[5], matches[RpmSpecBuiltRPMRegexArchitectureIndex])
+}
+
+func TestRpmSpecBuiltRPMRegexUnderscore(t *testing.T) {
+	input := "pkg-1.2.3-1_2.3.azl3.x86_64"
+	expectedResults := []string{"pkg", "", "1.2.3", "1_2.3", "azl3", "x86_64"}
+
+	matches := RpmSpecBuiltRPMRegex.FindStringSubmatch(input)
+	assert.Equal(t, RpmSpecBuiltRPMRegexMatchesCount, len(matches))
+
+	assert.Equal(t, expectedResults[0], matches[RpmSpecBuiltRPMRegexNameIndex])
+	assert.Equal(t, expectedResults[1], matches[RpmSpecBuiltRPMRegexEpochIndex])
+	assert.Equal(t, expectedResults[2], matches[RpmSpecBuiltRPMRegexVersionIndex])
+	assert.Equal(t, expectedResults[3], matches[RpmSpecBuiltRPMRegexReleaseIndex])
+	assert.Equal(t, expectedResults[4], matches[RpmSpecBuiltRPMRegexDistributionIndex])
+	assert.Equal(t, expectedResults[5], matches[RpmSpecBuiltRPMRegexArchitectureIndex])
+}
+
+func TestRpmSpecBuiltRPMRegexEpoch(t *testing.T) {
+	input := "ca-certificates-1:3.0.0-14.azl3.noarch"
+	expectedResults := []string{"ca-certificates", "1", "3.0.0", "14", "azl3", "noarch"}
+
+	matches := RpmSpecBuiltRPMRegex.FindStringSubmatch(input)
+	assert.Equal(t, RpmSpecBuiltRPMRegexMatchesCount, len(matches))
+
+	assert.Equal(t, expectedResults[0], matches[RpmSpecBuiltRPMRegexNameIndex])
+	assert.Equal(t, expectedResults[1], matches[RpmSpecBuiltRPMRegexEpochIndex])
+	assert.Equal(t, expectedResults[2], matches[RpmSpecBuiltRPMRegexVersionIndex])
+	assert.Equal(t, expectedResults[3], matches[RpmSpecBuiltRPMRegexReleaseIndex])
+	assert.Equal(t, expectedResults[4], matches[RpmSpecBuiltRPMRegexDistributionIndex])
+	assert.Equal(t, expectedResults[5], matches[RpmSpecBuiltRPMRegexArchitectureIndex])
+}

toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go

@@ -8,7 +8,6 @@ package rpmssnapshot
 import (
 	"fmt"
 	"path/filepath"
-	"regexp"
 	"runtime"
 
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/file"
@@ -23,37 +22,6 @@ const (
 	chrootOutputFilePath = "/snapshot.json"
 )
 
-// Regular expression to extract package name, version, distribution, and architecture from values returned by 'rpmspec --builtrpms'.
-// Examples:
-//
-//	kernel-5.15.63.1-1.azl3.x86_64		->	Name: kernel, Version: 5.15.63.1-1, Distribution: azl3, Architecture: x86_64
-//	python3-perf-5.15.63.1-1.azl3.x86_64	->	Name: python3-perf, Version: 5.15.63.1-1, Distribution: azl3, Architecture: x86_64
-//
-// NOTE: regular expression based on following assumptions:
-//   - Package version and release values are not allowed to contain a hyphen character.
-//   - Our tooling prevents the 'Release' tag from having any other form than '[[:digit:]]+%{?dist}'
-//   - The distribution tag is not allowed to contain a period or a hyphen.
-//   - The architecture is not allowed to contain a period or a hyphen.
-//
-// Regex breakdown:
-//
-//	^(.*)			<-- [index 1] package name
-//	-				<-- second-to-last hyphen separating the package name from its version
-//	([^-]+-[^-]+)		<-- [index 2] package version and package release number connected by the last hyphen
-//	\.				<-- second-to-last period separating the package release number from the distribution tag
-//	([^.]+)			<-- [index 3] the distribution tag
-//	\.				<-- last period separating the distribution tag from the architecture string
-//	([^.]+)$		<-- [index 4] the architecture string
-var rpmSpecBuiltRPMRegex = regexp.MustCompile(`^(.*)-([^-]+-[^-]+)\.([^.]+)\.([^.]+)$`)
-
-const (
-	rpmSpecBuiltRPMRegexNameIndex = iota + 1
-	rpmSpecBuiltRPMRegexVersionIndex
-	rpmSpecBuiltRPMRegexDistributionIndex
-	rpmSpecBuiltRPMRegexArchitectureIndex
-	rpmSpecBuiltRPMRegexMatchesCount
-)
-
 type SnapshotGenerator struct {
 	simpleToolChroot simpletoolchroot.SimpleToolChroot
 }
@@ -96,16 +64,24 @@ func (s *SnapshotGenerator) convertResultsToRepoContents(allBuiltRPMs []string)
 	}
 
 	for _, builtRPM := range allBuiltRPMs {
-		matches := rpmSpecBuiltRPMRegex.FindStringSubmatch(builtRPM)
-		if len(matches) != rpmSpecBuiltRPMRegexMatchesCount {
-			return repoContents, fmt.Errorf("RPM package name (%s) doesn't match the regular expression (%s)", builtRPM, rpmSpecBuiltRPMRegex.String())
+		matches := rpm.RpmSpecBuiltRPMRegex.FindStringSubmatch(builtRPM)
+		if len(matches) != rpm.RpmSpecBuiltRPMRegexMatchesCount {
+			return repoContents, fmt.Errorf("RPM package name (%s) doesn't match the regular expression (%s)", builtRPM, rpm.RpmSpecBuiltRPMRegex.String())
+		}
+
+		// Reattach a non-zero epoch to the version (`epoch:version-release`) so the resulting
+		// 'Version' field keeps the same shape it had before epoch was split out into its own
+		// capture group.
+		version := fmt.Sprintf("%s-%s", matches[rpm.RpmSpecBuiltRPMRegexVersionIndex], matches[rpm.RpmSpecBuiltRPMRegexReleaseIndex])
+		if epoch := matches[rpm.RpmSpecBuiltRPMRegexEpochIndex]; epoch != "" {
+			version = fmt.Sprintf("%s:%s", epoch, version)
 		}
 
 		repoContents.Repo = append(repoContents.Repo, &repocloner.RepoPackage{
-			Name:         matches[rpmSpecBuiltRPMRegexNameIndex],
-			Version:      matches[rpmSpecBuiltRPMRegexVersionIndex],
-			Distribution: matches[rpmSpecBuiltRPMRegexDistributionIndex],
-			Architecture: matches[rpmSpecBuiltRPMRegexArchitectureIndex],
+			Name:         matches[rpm.RpmSpecBuiltRPMRegexNameIndex],
+			Version:      version,
+			Distribution: matches[rpm.RpmSpecBuiltRPMRegexDistributionIndex],
+			Architecture: matches[rpm.RpmSpecBuiltRPMRegexArchitectureIndex],
 		})
 	}
 

toolkit/tools/pkg/rpmssnapshot/rpmssnapshot_test.go

@@ -9,41 +9,15 @@ import (
 	"testing"
 
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/packagerepo/repocloner"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/rpm"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestRegexBasic(t *testing.T) {
-	input := "pkg-1.2.3-1.azl3.x86_64"
-	expectedResults := []string{"pkg", "1.2.3-1", "azl3", "x86_64"}
-	expectedMatchNumber := 5
-
-	matches := rpmSpecBuiltRPMRegex.FindStringSubmatch(input)
-	assert.Equal(t, expectedMatchNumber, len(matches))
-
-	assert.Equal(t, expectedResults[0], matches[rpmSpecBuiltRPMRegexNameIndex])
-	assert.Equal(t, expectedResults[1], matches[rpmSpecBuiltRPMRegexVersionIndex])
-	assert.Equal(t, expectedResults[2], matches[rpmSpecBuiltRPMRegexDistributionIndex])
-	assert.Equal(t, expectedResults[3], matches[rpmSpecBuiltRPMRegexArchitectureIndex])
-}
-
-func TestRegexUnderscore(t *testing.T) {
-	input := "pkg-1.2.3-1_2.3.azl3.x86_64"
-	expectedResults := []string{"pkg", "1.2.3-1_2.3", "azl3", "x86_64"}
-	expectedMatchNumber := 5
-
-	matches := rpmSpecBuiltRPMRegex.FindStringSubmatch(input)
-	assert.Equal(t, expectedMatchNumber, len(matches))
-
-	assert.Equal(t, expectedResults[0], matches[rpmSpecBuiltRPMRegexNameIndex])
-	assert.Equal(t, expectedResults[1], matches[rpmSpecBuiltRPMRegexVersionIndex])
-	assert.Equal(t, expectedResults[2], matches[rpmSpecBuiltRPMRegexDistributionIndex])
-	assert.Equal(t, expectedResults[3], matches[rpmSpecBuiltRPMRegexArchitectureIndex])
-}
-
 func TestGenerateResults(t *testing.T) {
 	input := []string{
 		"pkg-1.2.3-1_2.3.azl3.x86_64",
 		"other-pkg-1.2.3-1_2.3.azl3.x86_64",
+		"ca-certificates-1:3.0.0-14.azl3.noarch",
 	}
 	expectedResults := repocloner.RepoContents{
 		Repo: []*repocloner.RepoPackage{
@@ -59,6 +33,12 @@ func TestGenerateResults(t *testing.T) {
 				Distribution: "azl3",
 				Architecture: "x86_64",
 			},
+			{
+				Name:         "ca-certificates",
+				Version:      "1:3.0.0-14",
+				Distribution: "azl3",
+				Architecture: "noarch",
+			},
 		},
 	}
 	emptySnapshotGenerator := SnapshotGenerator{}
@@ -73,5 +53,5 @@ func TestGenerateInvalidInput(t *testing.T) {
 	}
 	emptySnapshotGenerator := SnapshotGenerator{}
 	_, err := emptySnapshotGenerator.convertResultsToRepoContents(input)
-	assert.EqualError(t, err, "RPM package name ("+input[0]+") doesn't match the regular expression ("+rpmSpecBuiltRPMRegex.String()+")")
+	assert.EqualError(t, err, "RPM package name ("+input[0]+") doesn't match the regular expression ("+rpm.RpmSpecBuiltRPMRegex.String()+")")
 }

toolkit/tools/versionsprocessor/versionsprocessor.go

@@ -12,8 +12,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
 	"runtime"
+	"sort"
 	"strings"
 
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/exe"
@@ -28,8 +28,6 @@ import (
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
-var packageVersionRegexp = regexp.MustCompile(`^[^:]+: (?:(.+):)?(.+)-(.+)$`)
-
 var (
 	app           = kingpin.New("versionsprocessor", "A tool to generate a macro file of all specs version and release")
 	specsDir      = exe.InputDirFlag(app, "Directory to scan for SPECS")
@@ -140,25 +138,21 @@ func main() {
 }
 
 func processSpecFile(specFile string, buildArch string, distTag string, macrosOutput []string) (newMacrosOutput []string, err error) {
-	// Get spec file version-release
-
 	specFileName := filepath.Base(specFile)
 
 	sourceDir := filepath.Dir(specFile)
 	defines := rpm.DefaultDistroDefines(false, distTag)
 
-	packages, err := rpm.QuerySPEC(specFile, sourceDir, `%{NAME}: %{evr}\n`, buildArch, defines, rpm.QueryHeaderArgument)
-
+	packages, err := rpm.QuerySPECForBuiltRPMs(specFile, sourceDir, buildArch, defines)
 	if err != nil {
 		logger.Log.Errorf("Failed to query spec file (%s). Error: %s", specFileName, err)
 		return nil, err
 	}
 
-	for _, packageVersionString := range packages {
-
-		macros, err := processPackageVersionString(packageVersionString, specFileName, distTag)
-		if err != nil {
-			logger.Log.Errorf("Error processing package version string: %s", err)
+	for _, packageNEVRA := range packages {
+		macros, processErr := processPackageVersionString(packageNEVRA)
+		if processErr != nil {
+			logger.Log.Errorf("Failed to process package (%s) from spec file (%s): %s", packageNEVRA, specFileName, processErr)
 			continue
 		}
 
@@ -168,44 +162,35 @@ func processSpecFile(specFile string, buildArch string, distTag string, macrosOu
 	return macrosOutput, nil
 }
 
-func processPackageVersionString(packageVersionString string, specFileName string, distTag string) (macros []string, err error) {
-	const (
-		prefix = "azl"
-	)
-	// the output of the above query is in the format of "packagename: version-release",
-	// so split by ": " to get the version-release portion we want the second part
-	releaseVerSplit := packageVersionRegexp.FindStringSubmatch(packageVersionString)[1:]
-
-	if len(releaseVerSplit) <= 2 {
-		errorString := fmt.Sprintf("Empty version-release format retrieved from spec file (%s)", specFileName)
-		err = fmt.Errorf(errorString)
-		logger.Log.Errorf(errorString)
+func processPackageVersionString(packageNEVRA string) (macros []string, err error) {
+	const prefix = "azl"
 
-		return []string{""}, err
+	matches := rpm.RpmSpecBuiltRPMRegex.FindStringSubmatch(packageNEVRA)
+	if len(matches) != rpm.RpmSpecBuiltRPMRegexMatchesCount {
+		return nil, fmt.Errorf("invalid package NEVRA format: %q", packageNEVRA)
 	}
 
-	epoch := releaseVerSplit[0]
-	version := releaseVerSplit[1]
-	release := releaseVerSplit[2]
-	releaseClean := strings.Replace(release, distTag, "", 1)
+	name := matches[rpm.RpmSpecBuiltRPMRegexNameIndex]
+	epoch := matches[rpm.RpmSpecBuiltRPMRegexEpochIndex]
+	version := matches[rpm.RpmSpecBuiltRPMRegexVersionIndex]
+	release := matches[rpm.RpmSpecBuiltRPMRegexReleaseIndex]
 
-	// strip out the .spec suffix and replace '-' with '_' as RPM macros cannot have '-'
-	packageFileNameMacroFormat := strings.Replace(specFileName, ".spec", "", 1)
-	packageFileNameMacroFormat = strings.ReplaceAll(packageFileNameMacroFormat, "-", "_")
+	// Replace '-' with '_' as RPM macros cannot have '-'.
+	nameMacroFormat := strings.ReplaceAll(name, "-", "_")
 
-	epochReleaseString := prefix + "_" + packageFileNameMacroFormat + "_epoch"
-	versionMacroString := prefix + "_" + packageFileNameMacroFormat + "_version"
-	releaseMacroString := prefix + "_" + packageFileNameMacroFormat + "_release"
+	epochMacroString := prefix + "_" + nameMacroFormat + "_epoch"
+	versionMacroString := prefix + "_" + nameMacroFormat + "_version"
+	releaseMacroString := prefix + "_" + nameMacroFormat + "_release"
 
 	// Generate RPM macro definitions instead of modifying spec files directly.
 	macros = []string{
 		fmt.Sprintf("%%%s %s", versionMacroString, version),
-		fmt.Sprintf("%%%s %s", releaseMacroString, releaseClean),
+		fmt.Sprintf("%%%s %s", releaseMacroString, release),
 	}
 
 	// Only append (to the front of the list) if we have an epoch
 	if epoch != "" {
-		macros = append([]string{fmt.Sprintf("%%%s %s", epochReleaseString, epoch)}, macros...)
+		macros = append([]string{fmt.Sprintf("%%%s %s", epochMacroString, epoch)}, macros...)
 	}
 
 	return macros, nil
@@ -218,16 +203,18 @@ func writeExtraFilesToOutput(extraFiles []string, macrosOutput []string, output
 			continue
 		}
 
-		contents, readErr := file.Read(extraPath)
+		contents, readErr := file.ReadLines(extraPath)
 		if readErr != nil {
 			logger.Log.Errorf("Failed to read extra macros file (%s): %s", extraPath, readErr)
 			continue
 		}
 
-		macrosOutput = append(macrosOutput, contents)
+		macrosOutput = append(macrosOutput, contents...)
 		logger.Log.Infof("Appended contents of provided extra macros file (%s) to %s", extraPath, output)
 	}
 
+	sort.Strings(macrosOutput)
+
 	err = file.WriteLines(macrosOutput, output)
 	if err != nil {
 		logger.Log.Errorf("Failed to write file (%s)", output)