perf(ci): cache used actions as part of act cache warmup (#495)

Author: xnyo

Makefile

@@ -1,4 +1,6 @@
-.PHONY: reset-mockdata clean-node-modules clean-act-tmp clean-lfs mockdata-dist mockdata-dist-artifacts mockdata
+.PHONY: clean-node-modules clean-dist clean-act-tmp clean-act-toolcache-volumes clean-lfs clean \
+	reset-mockdata mockdata-dist mockdata-dist-artifacts mockdata \
+	genreadme act-lint act-test actionlint
 
 clean-node-modules:
 	find tests -name node_modules -type d -prune -exec rm -rf '{}' +
@@ -8,7 +10,7 @@ clean-dist:
 
 clean-act-tmp:
 	rm -rf /tmp/act-artifacts
-	rm -rf /tmp/act-action-cache
+	rm -rf /tmp/act-actions-cache
 	rm -rf /tmp/act-cache
 	rm -rf /tmp/act-gcs
 

tests/act/internal/act/act.go

@@ -6,9 +6,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
@@ -43,6 +45,10 @@ type Runner struct {
 	// uuid is a unique identifier for this Runner instance.
 	uuid uuid.UUID
 
+	// actionsCachePath is the absolute path where GitHub Actions are cached.
+	// If empty, a new temporary directory is created for each runner.
+	actionsCachePath string
+
 	// ArtifactsStorage is the storage for artifacts uploaded during the workflow run.
 	ArtifactsStorage ArtifactsStorage
 
@@ -65,6 +71,13 @@ type Runner struct {
 	ContainerArchitecture string
 }
 
+// actionsCachePathBase is the base path for the action cache.
+var actionsCachePathBase = filepath.Join("/tmp", "act-actions-cache")
+
+// TemplateActionsCachePath is the path where GitHub Actions are pre-cached during the warmup workflow.
+// This cache is then copied to each runner's actions cache path for the actual workflow runs.
+var TemplateActionsCachePath = filepath.Join(actionsCachePathBase, "template")
+
 // RunnerOption is a function that configures a Runner.
 type RunnerOption func(r *Runner)
 
@@ -88,6 +101,13 @@ func WithLinuxAMD64ContainerArchitecture() RunnerOption {
 	return WithContainerArchitecture("linux/amd64")
 }
 
+// WithActionsCachePath sets the actions cache path for the runner (absolute path).
+func WithActionsCachePath(cachePath string) RunnerOption {
+	return func(r *Runner) {
+		r.actionsCachePath = cachePath
+	}
+}
+
 // NewRunner creates a new Runner instance.
 func NewRunner(t *testing.T, opts ...RunnerOption) (*Runner, error) {
 	// Get GitHub token from environment (GHA) or gh CLI (local)
@@ -119,7 +139,10 @@ func NewRunner(t *testing.T, opts ...RunnerOption) (*Runner, error) {
 	for _, opt := range opts {
 		opt(r)
 	}
-
+	// Default to a new temporary directory for the actions cache if not set.
+	if r.actionsCachePath == "" {
+		WithActionsCachePath(filepath.Join(actionsCachePathBase, r.uuid.String()))(r)
+	}
 	return r, nil
 }
 
@@ -143,9 +166,6 @@ func (r *Runner) args(eventKind EventKind, actor string, workflowFile string, pa
 		// Unique artifact server port and path per act runner instance
 		fmt.Sprintf("--artifact-server-port=%d", artifactServerPort),
 		"--artifact-server-path=/tmp/act-artifacts/" + r.uuid.String() + "/",
-		// Unique action cache path per act runner instance to prevent cache corruption
-		// when running multiple tests in parallel or reusing runners with stale cache
-		"--action-cache-path=/tmp/act-action-cache/" + r.uuid.String() + "/",
 
 		// Required for cloning private repos
 		"--secret", "GITHUB_TOKEN=" + r.gitHubToken,
@@ -156,6 +176,19 @@ func (r *Runner) args(eventKind EventKind, actor string, workflowFile string, pa
 		// - /tmp: for temporary files, so the host's /tmp is used
 		"--container-options", `"-v $PWD/tests/act/mockdata:/mockdata -v ` + r.GCS.basePath + `:/gcs -v /tmp:/tmp"`,
 	}
+	if r.actionsCachePath != "" {
+		// Create and use per-runner cache.
+		// Do not pre-populate the cache if we are using the shared cache (cache warmup).
+		if r.actionsCachePath != TemplateActionsCachePath {
+			if err := copyDir(TemplateActionsCachePath, r.actionsCachePath); err != nil {
+				return nil, fmt.Errorf("copy action cache: %w", err)
+			}
+		}
+		args = append(args, "--action-cache-path", r.actionsCachePath)
+	} else {
+		// Use shared cache
+		args = append(args, "--action-cache-path", TemplateActionsCachePath)
+	}
 
 	// Map local all possible references of plugin-ci-workflows to the local repository
 	localRepoArgs, err := r.localRepositoryArgs()
@@ -288,7 +321,10 @@ func (r *Runner) Run(workflow workflow.Workflow, event Event) (runResult *RunRes
 		return nil, fmt.Errorf("start act: %w", err)
 	}
 
-	// Process json logs in stdout stream
+	// Process json logs in stdout stream.
+	// This must complete BEFORE cmd.Wait() is called, because cmd.Wait()
+	// closes the stdout pipe. If the goroutine is still reading when the
+	// pipe is closed, it will get a "file already closed" error.
 	errs := make(chan error, 1)
 	go func() {
 		if err := r.processStream(stdout, runResult); err != nil {
@@ -297,7 +333,15 @@ func (r *Runner) Run(workflow workflow.Workflow, event Event) (runResult *RunRes
 		errs <- nil
 	}()
 
-	// Wait for act to finish
+	// Wait for stdout processing to complete FIRST.
+	// The scanner will return EOF when the process exits and closes its stdout.
+	if streamErr := <-errs; streamErr != nil {
+		// Still call Wait to clean up the process
+		_ = cmd.Wait()
+		return nil, streamErr
+	}
+
+	// Now wait for the process to fully exit and get its exit status.
 	if err := cmd.Wait(); err != nil {
 		if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() != 0 {
 			runResult.Success = false
@@ -307,8 +351,7 @@ func (r *Runner) Run(workflow workflow.Workflow, event Event) (runResult *RunRes
 	}
 	runResult.Success = true
 
-	// Wait for stdout processing to complete
-	return runResult, <-errs
+	return runResult, nil
 }
 
 // processStream processes the given reader line by line as JSON log lines generated by act.
@@ -491,3 +534,84 @@ func getFreePort() (port int, err error) {
 	}
 	return
 }
+
+// copyDir recursively copies a directory tree from src to dst.
+// If src does not exist, it creates an empty dst directory.
+func copyDir(src, dst string) error {
+	// Check if source exists
+	srcInfo, err := os.Stat(src)
+	if os.IsNotExist(err) {
+		// Source doesn't exist, just create an empty destination directory
+		return os.MkdirAll(dst, 0755)
+	}
+	if err != nil {
+		return fmt.Errorf("stat source: %w", err)
+	}
+	if !srcInfo.IsDir() {
+		return fmt.Errorf("source is not a directory: %s", src)
+	}
+
+	// Create destination directory
+	if err := os.MkdirAll(dst, srcInfo.Mode()); err != nil {
+		return fmt.Errorf("create destination directory: %w", err)
+	}
+
+	// Walk through source directory and copy all files
+	return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Calculate relative path and destination path
+		relPath, err := filepath.Rel(src, path)
+		if err != nil {
+			return fmt.Errorf("get relative path: %w", err)
+		}
+		dstPath := filepath.Join(dst, relPath)
+
+		if d.IsDir() {
+			info, err := d.Info()
+			if err != nil {
+				return fmt.Errorf("get dir info: %w", err)
+			}
+			return os.MkdirAll(dstPath, info.Mode())
+		}
+
+		// Copy file
+		return copyFile(path, dstPath)
+	})
+}
+
+// copyFile copies a single file from src to dst, preserving permissions.
+func copyFile(src, dst string) (err error) {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("open source file: %w", err)
+	}
+	defer func() {
+		if closeErr := srcFile.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+	}()
+
+	srcInfo, err := srcFile.Stat()
+	if err != nil {
+		return fmt.Errorf("stat source file: %w", err)
+	}
+
+	dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, srcInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("create destination file: %w", err)
+	}
+	defer func() {
+		if closeErr := dstFile.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+	}()
+
+	if _, err := io.Copy(dstFile, srcFile); err != nil {
+		return fmt.Errorf("copy file contents: %w", err)
+	}
+
+	return nil
+}

tests/act/internal/act/actions.go

@@ -0,0 +1,69 @@
+package act
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// actionUsesRegex matches "uses: <action>@<ref>" patterns in YAML files.
+// It captures the full action reference including the version/commit.
+var actionUsesRegex = regexp.MustCompile(`uses:\s*["']?([^@\s"'#]+@[^\s"'#]+)`)
+
+// ExtractExternalActions scans YAML files in the given directories and returns
+// unique external action references (uses: owner/repo@ref).
+// It filters out internal grafana/plugin-ci-workflows references.
+// The returned slice contains deduplicated action references sorted alphabetically.
+func ExtractExternalActions(dirs ...string) ([]string, error) {
+	seen := make(map[string]struct{})
+	var actions []string
+
+	for _, dir := range dirs {
+		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() {
+				return nil
+			}
+			// Only process YAML files
+			ext := filepath.Ext(path)
+			if ext != ".yml" && ext != ".yaml" {
+				return nil
+			}
+
+			content, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+
+			matches := actionUsesRegex.FindAllSubmatch(content, -1)
+			for _, match := range matches {
+				if len(match) < 2 {
+					continue
+				}
+				action := string(match[1])
+
+				// Skip internal references
+				if strings.HasPrefix(action, "grafana/plugin-ci-workflows") {
+					continue
+				}
+
+				// Skip if already seen
+				if _, ok := seen[action]; ok {
+					continue
+				}
+				seen[action] = struct{}{}
+				actions = append(actions, action)
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return actions, nil
+}
+