test: add tests for workflows consistency (#568)

Author: xnyo

.github/workflows/release-please-pr-update-tagged-references.yml

@@ -47,7 +47,8 @@ jobs:
 
   update-tagged-references:
     name: Update tagged references
-    needs: automated-commit-check
+    needs:
+      - automated-commit-check
     permissions:
       contents: write
       pull-requests: write

tests/act/main_workflow_consistency_test.go

@@ -0,0 +1,241 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/grafana/plugin-ci-workflows/tests/act/internal/workflow"
+	"github.com/stretchr/testify/require"
+)
+
+type workflowEntry struct {
+	path     string
+	internal bool
+}
+
+// knownWorkflows is the list of all workflow files in .github/workflows/.
+// Each entry is classified as public (used by plugin repos) or internal (used by this repo only).
+// If a new workflow is added to the repo, it must be added here or the tests will fail.
+var knownWorkflows = []workflowEntry{
+	// Public workflows (used by plugin repos)
+	{path: "ci.yml", internal: false},
+	{path: "cd.yml", internal: false},
+	{path: "playwright.yml", internal: false},
+	{path: "playwright-docker.yml", internal: false},
+	{path: "check-release-channel.yml", internal: false},
+
+	// Internal workflows (used by this repo only)
+	{path: "pr-checks-examples-readmes.yml", internal: true},
+	{path: "pr-checks-lint.yml", internal: true},
+	{path: "pr-checks-semantic-title.yml", internal: true},
+	{path: "pr-checks-test-ci.yml", internal: true},
+	{path: "pr-checks-workflow-references.yml", internal: true},
+	{path: "release-please-pr-update-tagged-references.yml", internal: true},
+	{path: "release-please-restore-rolling-release.yml", internal: true},
+	{path: "release-please.yml", internal: true},
+}
+
+// ciOnlyInputs lists ci.yml inputs that are CI-only and should NOT be in cd.yml.
+var ciOnlyInputs = map[string]bool{
+	"testing": true,
+}
+
+func TestCDWorkflowContainsAllCIInputs(t *testing.T) {
+	// Ensure that all inputs defined in ci.yml are also defined in cd.yml, so that CD can pass them through to CI.
+	t.Parallel()
+
+	ciWf, err := workflow.NewBaseWorkflowFromFile(filepath.Join(".github", "workflows", "ci.yml"))
+	require.NoError(t, err)
+	cdWf, err := workflow.NewBaseWorkflowFromFile(filepath.Join(".github", "workflows", "cd.yml"))
+	require.NoError(t, err)
+
+	ciInputs := ciWf.On.WorkflowCall.Inputs
+	cdInputs := cdWf.On.WorkflowCall.Inputs
+
+	for inputName := range ciInputs {
+		if ciOnlyInputs[inputName] {
+			continue
+		}
+		require.Contains(t, cdInputs, inputName, "cd.yml should contain ci.yml input %q", inputName)
+	}
+}
+
+func TestCDWorkflowCIJobPassesAllInputs(t *testing.T) {
+	// Ensure that CD's ci job passes all relevant inputs to CI, so that CI can function correctly when called from CD.
+	t.Parallel()
+
+	ciWf, err := workflow.NewBaseWorkflowFromFile(filepath.Join(".github", "workflows", "ci.yml"))
+	require.NoError(t, err)
+	cdWf, err := workflow.NewBaseWorkflowFromFile(filepath.Join(".github", "workflows", "cd.yml"))
+	require.NoError(t, err)
+
+	ciInputs := ciWf.On.WorkflowCall.Inputs
+
+	cdCIJob := cdWf.Jobs["ci"]
+	require.NotNil(t, cdCIJob, "cd.yml should have a 'ci' job")
+	cdCIJobWith := cdCIJob.With
+
+	// Forward: every CI input should be passed by CD's ci job
+	t.Run("ci inputs are passed", func(t *testing.T) {
+		t.Parallel()
+		for inputName := range ciInputs {
+			if ciOnlyInputs[inputName] {
+				continue
+			}
+			require.Contains(t, cdCIJobWith, inputName, "cd.yml ci job should pass ci.yml input %q", inputName)
+		}
+	})
+
+	// Reverse: every key in CD's ci job 'with' should be a valid CI input
+	t.Run("cd ci with keys are valid", func(t *testing.T) {
+		for withKey := range cdCIJobWith {
+			require.Contains(t, ciInputs, withKey, "cd.yml ci job passes unknown input %q to ci.yml", withKey)
+		}
+	})
+}
+
+func TestExportedWorkflowsInSyncWithReleasePlease(t *testing.T) {
+	t.Parallel()
+
+	knownSet := make(map[string]struct{}, len(knownWorkflows))
+	for _, w := range knownWorkflows {
+		knownSet[w.path] = struct{}{}
+	}
+
+	var publicPaths []string
+	for _, w := range knownWorkflows {
+		if !w.internal {
+			publicPaths = append(publicPaths, filepath.Join(".github", "workflows", w.path))
+		}
+	}
+
+	t.Run("all workflows are classified", func(t *testing.T) {
+		t.Parallel()
+
+		files, err := filepath.Glob(filepath.Join(".github", "workflows", "*.yml"))
+		require.NoError(t, err)
+
+		for _, f := range files {
+			name := filepath.Base(f)
+			// Skip temporary act workflow files generated by the test framework
+			if strings.HasPrefix(name, "act-") {
+				continue
+			}
+			require.Contains(t, knownSet, name, "workflow file %q is not classified as public or internal; please add it to knownWorkflows", name)
+		}
+	})
+
+	type switchRefSource struct {
+		name   string
+		file   string
+		job    string
+		stepID string
+	}
+
+	sources := []switchRefSource{
+		{
+			name:   "pr-checks-workflow-references",
+			file:   "pr-checks-workflow-references.yml",
+			job:    "check-references",
+			stepID: "switch-references",
+		},
+		{
+			name:   "release-please-pr-update-tagged-references",
+			file:   "release-please-pr-update-tagged-references.yml",
+			job:    "update-tagged-references",
+			stepID: "switch-references",
+		},
+		{
+			name:   "release-please-restore-rolling-release",
+			file:   "release-please-restore-rolling-release.yml",
+			job:    "update-tagged-references",
+			stepID: "switch-references",
+		},
+	}
+
+	for _, src := range sources {
+		t.Run(src.name, func(t *testing.T) {
+			t.Parallel()
+
+			wf, err := workflow.NewBaseWorkflowFromFile(filepath.Join(".github", "workflows", src.file))
+			require.NoError(t, err)
+
+			job := wf.Jobs[src.job]
+			require.NotNilf(t, job, "job %q not found in %s", src.job, src.file)
+
+			step := job.GetStep(src.stepID)
+			require.NotNilf(t, step, "step %q not found in job %q of %s", src.stepID, src.job, src.file)
+
+			pathsRaw, ok := step.With["paths"]
+			require.True(t, ok, "step %q in job %q of %s should have 'paths' in with", src.stepID, src.job, src.file)
+
+			pathsStr, ok := pathsRaw.(string)
+			require.True(t, ok, "'paths' should be a string, got %T", pathsRaw)
+
+			var paths []string
+			for _, line := range strings.Split(pathsStr, "\n") {
+				trimmed := strings.TrimSpace(line)
+				if trimmed != "" {
+					paths = append(paths, trimmed)
+				}
+			}
+
+			require.Contains(t, paths, "actions/plugins/**", "%s should contain 'actions/plugins/**' in switch-references paths", src.file)
+			for _, publicPath := range publicPaths {
+				require.Contains(t, paths, publicPath, "%s switch-references paths should contain public workflow %q", src.file, publicPath)
+			}
+		})
+	}
+}
+
+func TestPluginActionsInReleasePleaseConfig(t *testing.T) {
+	// Ensure that all public plugin actions (those not under actions/internal/) are registered
+	// in release-please-config.json's "packages" section.
+	t.Parallel()
+
+	// Discover all action directories under actions/ that are NOT under actions/internal/.
+	// An action directory is one that contains action.yml or action.yaml.
+	actionDirs, err := findPluginActionDirs("actions")
+	require.NoError(t, err)
+	require.NotEmpty(t, actionDirs, "should find at least one plugin action")
+
+	// Parse release-please-config.json to get the packages map.
+	configBytes, err := os.ReadFile("release-please-config.json")
+	require.NoError(t, err)
+
+	var config struct {
+		Packages map[string]json.RawMessage `json:"packages"`
+	}
+	require.NoError(t, json.Unmarshal(configBytes, &config))
+
+	for _, actionDir := range actionDirs {
+		require.Contains(t, config.Packages, actionDir, "action %q should be listed in release-please-config.json packages", actionDir)
+	}
+}
+
+// findPluginActionDirs walks the given root and returns directories that contain
+// an action.yml or action.yaml file, excluding anything under actions/internal/.
+func findPluginActionDirs(root string) ([]string, error) {
+	var dirs []string
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// Skip the actions/internal subtree entirely.
+		if info.IsDir() && path == filepath.Join("actions", "internal") {
+			return filepath.SkipDir
+		}
+		if info.IsDir() {
+			return nil
+		}
+		base := filepath.Base(path)
+		if base == "action.yml" || base == "action.yaml" {
+			dirs = append(dirs, filepath.Dir(path))
+		}
+		return nil
+	})
+	return dirs, err
+}