circleci: fix cache save by eliminating fan-in workspace collision

Problem: CircleCI throws 'Concurrent upstream jobs persisted the same
file(s) into the workspace' when multiple jobs try to persist the
same paths (packages/x86, build/x86, etc.) simultaneously.

Previous approach: Used separate save_cache_x86/save_cache_ppc64 jobs that
attached multiple upstream workspaces, but still hit the fan-in collision.

Fix: Each seed job saves its cache immediately after building, before
persisting to workspace. No fan-in jobs needed.

New job types:
- build_and_persist_save_cache_x86: x86-musl-cross-make only, saves musl cache
- build_and_persist_save_cache_coreboot_x86: per-fork seeds, saves fork-specific
  coreboot+toolchain cache keyed with -<coreboot_dir> suffix
- build_and_persist_save_cache_ppc64: talos-2 builds toolchain+board, saves all

Cache restore: Added per-fork fallback keys in prep_env to match the new
fork-specific cache key format (e.g., ...-coreboot-dasharo_msi_z790).

Each coreboot fork now has its own cache:
- coreboot-dasharo_nv4x (novacustom-nv4x_adl)
- coreboot-dasharo_v56 (novacustom-v560tu)
- coreboot-purism (librem_14)
- coreboot-25.09 (EOL_t480-hotp-maximized)
- coreboot-4.11 (EOL_librem_l1um)
- coreboot-dasharo_msi_z690 (UNTESTED_msi_z690a_ddr4)
- coreboot-dasharo_msi_z790 (UNTESTED_msi_z790p_ddr4)

Last fork (UNTESTED_msi_z790p_ddr4) also saves full modules cache.

Signed-off-by: Thierry Laurion <insurgo@riseup.net>

Author: tlaurion

.circleci/config.yml

@@ -75,16 +75,19 @@ jobs:
           command: |
             find ./flake.lock modules/musl-cross-make* -type f | sort -h | xargs sha256sum > ./tmpDir/musl-cross-make.sha256sums
       - restore_cache:
-          # First matched/found key wins and following keys are not tried
           keys:
-            # Cache for matching modules digest, validated to be exactly the same as in GitHub current commit.
-            # This cache was made on top of below caches, if previously existing.
-            # If no module definition changed, we reuse this one
+            # First matched/found key wins and following keys are not tried.
             # x86 caches
             - nix-docker-heads-modules-x86-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
-            - nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+            # coreboot-musl cache: exact match + fork fallbacks
+            - nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}-coreboot-dasharo_msi_z790
+            - nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}-coreboot-dasharo_msi_z690
+            - nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}-coreboot-4.11
+            - nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}-coreboot-25.09
+            - nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}-coreboot-purism
+            - nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}-coreboot-dasharo_v56
+            - nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}-coreboot-dasharo_nv4x
             - nix-docker-heads-musl-cross-make-x86-{{ checksum "./tmpDir/musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}
-            # Cache for coreboot module (and patches) and musl-cross-make digests (coreboot: triannual release)
             # ppc64 caches
             - nix-docker-heads-modules-ppc64-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
             - nix-docker-heads-coreboot-musl-cross-make-ppc64-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}
@@ -175,78 +178,137 @@ jobs:
           target: <<parameters.target>>
           subcommand: <<parameters.subcommand>>
 
-  save_cache_x86:
+  # build_and_persist_save_cache_x86: builds musl-cross-make for x86 and saves the
+  # musl-cross-make cache immediately. Used only for x86-musl-cross-make.
+  # Workspace chain: prep_env -> here (linear, no collision).
+  build_and_persist_save_cache_x86:
     docker:
-      # Docker image: tlaurion/heads-dev-env:v0.2.9
       - image: tlaurion/heads-dev-env@sha256:96f8f91c6464305c4a990d59f9ef93910c16c7fd0501a46b43b34a4600a368de
     resource_class: large
     working_directory: ~/heads
+    parameters:
+      target:
+        type: string
+      subcommand:
+        type: string
     steps:
       - attach_workspace:
           at: ~/heads
-      - save_cache:
-          # Generate cache for the same musl-cross-make module definition if hash is not previously existing
-          # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names
-          key: nix-docker-heads-musl-cross-make-x86-{{ checksum "./tmpDir/musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+      - build_board:
+          arch: x86
+          target: << parameters.target >>
+          subcommand: << parameters.subcommand >>
+      - persist_to_workspace:
+          root: ~/heads
           paths:
-            - build/x86/musl-cross-make-fd6be58297ee21fcba89216ccd0d4aca1e3f1c5c
-            - crossgcc/x86
             - packages/x86
+            - build/x86
+            - crossgcc/x86
+            - install/x86
       - save_cache:
-          # Generate cache for the same coreboot and musl-cross-make modules definition if hash is not previously existing
-          # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names
-          key: nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+          key: nix-docker-heads-musl-cross-make-x86-{{ checksum "./tmpDir/musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}
           paths:
-            - build/x86/coreboot-4.11
-            - build/x86/coreboot-24.02.01
-            - build/x86/coreboot-25.09
-            - build/x86/coreboot-dasharo_v56
-            - build/x86/coreboot-dasharo_nv4x
-            - build/x86/coreboot-dasharo_msi_z790
-            - build/x86/coreboot-dasharo_msi_z690
-            - build/x86/coreboot-purism
             - build/x86/musl-cross-make-fd6be58297ee21fcba89216ccd0d4aca1e3f1c5c
             - crossgcc/x86
             - packages/x86
-      - save_cache:
-          # Generate cache for the exact same modules definitions if hash is not previously existing
-          key: nix-docker-heads-modules-x86-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+
+  # build_and_persist_save_cache_coreboot_x86: builds a coreboot fork seed board and
+  # saves its fork-specific coreboot+toolchain cache immediately after building.
+  # Used for each distinct coreboot fork (24.02.01, purism, 25.09, 4.11, dasharo_*).
+  # Workspace chain at this point: prep_env -> x86-musl-cross-make -> here.
+  # Still linear — no concurrent upstream collision.
+  # Each fork saves only the coreboot dirs it built, keyed on coreboot+musl digest.
+  # The full modules cache (nix-docker-heads-modules-x86) is saved by the last fork.
+  build_and_persist_save_cache_coreboot_x86:
+    docker:
+      - image: tlaurion/heads-dev-env@sha256:96f8f91c6464305c4a990d59f9ef93910c16c7fd0501a46b43b34a4600a368de
+    resource_class: large
+    working_directory: ~/heads
+    parameters:
+      target:
+        type: string
+      subcommand:
+        type: string
+      coreboot_dir:
+        type: string
+        description: "The build/x86/coreboot-* directory built by this fork (e.g. coreboot-24.02.01)"
+      save_full_modules_cache:
+        type: boolean
+        default: false
+        description: "Set true only on the last coreboot fork in sequence to save the full modules cache"
+    steps:
+      - attach_workspace:
+          at: ~/heads
+      - build_board:
+          arch: x86
+          target: << parameters.target >>
+          subcommand: << parameters.subcommand >>
+      - persist_to_workspace:
+          root: ~/heads
           paths:
+            - packages/x86
             - build/x86
+            - crossgcc/x86
             - install/x86
+      - save_cache:
+          key: nix-docker-heads-coreboot-musl-cross-make-x86-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}-<< parameters.coreboot_dir >>
+          paths:
+            - build/x86/<< parameters.coreboot_dir >>
+            - build/x86/musl-cross-make-fd6be58297ee21fcba89216ccd0d4aca1e3f1c5c
             - crossgcc/x86
             - packages/x86
-
-
-
-  save_cache_ppc64:
+      - when:
+          condition: << parameters.save_full_modules_cache >>
+          steps:
+            - save_cache:
+                key: nix-docker-heads-modules-x86-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
+                paths:
+                  - build/x86
+                  - install/x86
+                  - crossgcc/x86
+                  - packages/x86
+
+  # build_and_persist_save_cache_ppc64: builds ppc64 toolchain and talos-2 board,
+  # saves all ppc64 caches immediately. ppc64 has only one coreboot fork, so no
+  # fan-in and no workspace collision.
+  build_and_persist_save_cache_ppc64:
     docker:
-      # Docker image: tlaurion/heads-dev-env:v0.2.9
       - image: tlaurion/heads-dev-env@sha256:96f8f91c6464305c4a990d59f9ef93910c16c7fd0501a46b43b34a4600a368de
     resource_class: large
     working_directory: ~/heads
+    parameters:
+      target:
+        type: string
+      subcommand:
+        type: string
     steps:
       - attach_workspace:
           at: ~/heads
+      - build_board:
+          arch: ppc64
+          target: << parameters.target >>
+          subcommand: << parameters.subcommand >>
+      - persist_to_workspace:
+          root: ~/heads
+          paths:
+            - packages/ppc64
+            - build/ppc64
+            - crossgcc/ppc64
+            - install/ppc64
       - save_cache:
-          # Generate cache for the same musl-cross-make module definition if hash is not previously existing
-          # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names
           key: nix-docker-heads-musl-cross-make-ppc64-{{ checksum "./tmpDir/musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}
           paths:
             - build/ppc64/musl-cross-make-fd6be58297ee21fcba89216ccd0d4aca1e3f1c5c
             - crossgcc/ppc64
             - packages/ppc64
       - save_cache:
-          # Generate cache for the same coreboot and musl-cross-make modules definition if hash is not previously existing
-          # CircleCI removed their wildcard support, so we have to list precise versions to cache in directory names
           key: nix-docker-heads-coreboot-musl-cross-make-ppc64-{{ checksum "./tmpDir/coreboot_musl-cross-make.sha256sums" }}{{ .Environment.CACHE_VERSION }}
           paths:
             - build/ppc64/coreboot-talos_2
             - build/ppc64/musl-cross-make-fd6be58297ee21fcba89216ccd0d4aca1e3f1c5c
             - crossgcc/ppc64
             - packages/ppc64
       - save_cache:
-          # Generate cache for the exact same modules definitions if hash is not previously existing
           key: nix-docker-heads-modules-ppc64-{{ checksum "./tmpDir/all_modules_and_patches.sha256sums" }}{{ .Environment.CACHE_VERSION }}
           paths:
             - build/ppc64
@@ -261,98 +323,94 @@ workflows:
     jobs:
       - prep_env
 
-      # This step builds musl-cross-make for x86 architecture, which will be used by subsequent x86 board builds
-      - build_and_persist:
+      # ── x86 toolchain seed ────────────────────────────────────────────────
+      # Builds musl-cross-make for x86. Saves musl-cross-make cache immediately.
+      # Chain: prep_env -> here (linear, no collision).
+      - build_and_persist_save_cache_x86:
           name: x86-musl-cross-make
           target: EOL_t480-hotp-maximized
           subcommand: "musl-cross-make"
           requires:
             - prep_env
 
-      # This step builds musl-cross-make for ppc64 architecture, which will be used by subsequent ppc64 board builds
-      - build_and_persist:
-          name: ppc64-musl-cross-make
-          arch: ppc64
+      # ── ppc64 toolchain + board ───────────────────────────────────────────
+      # Builds musl-cross-make then talos-2 board. Saves all ppc64 caches.
+      # Chain: prep_env -> ppc64-musl-cross-make -> UNTESTED_talos-2 (linear, no collision).
+      - build_and_persist_save_cache_ppc64:
+          name: UNTESTED_talos-2
           target: UNTESTED_talos-2
-          subcommand: "musl-cross-make"
+          subcommand: ""
           requires:
             - prep_env
 
-      # Below, sequentially build one board for each coreboot version.
-      # The last board in the sequence is the dependency for the parallel boards built at the end, and also save_cache.
+      # ── coreboot fork seeds (x86) ─────────────────────────────────────────
+      # Each seed builds exactly one coreboot fork toolchain and saves its own
+      # fork-specific cache keyed on coreboot_dir. No fan-in -> no workspace collision.
+      # Chain for all: prep_env -> x86-musl-cross-make -> here (linear)
 
-      # coreboot 24.02.01
-      - build_and_persist:
+      # coreboot-dasharo_nv4x
+      - build_and_persist_save_cache_coreboot_x86:
           name: novacustom-nv4x_adl
           target: novacustom-nv4x_adl
           subcommand: ""
+          coreboot_dir: coreboot-dasharo_nv4x
           requires:
             - x86-musl-cross-make
 
-      # coreboot purism: builds 24.02.01 toolchain, all librem boards share it
-      - build_and_persist:
+      # coreboot-dasharo_v56
+      - build_and_persist_save_cache_coreboot_x86:
+          name: novacustom-v560tu
+          target: novacustom-v560tu
+          subcommand: ""
+          coreboot_dir: coreboot-dasharo_v56
+          requires:
+            - x86-musl-cross-make
+
+      # coreboot-purism
+      - build_and_persist_save_cache_coreboot_x86:
           name: librem_14
           target: librem_14
           subcommand: ""
+          coreboot_dir: coreboot-purism
           requires:
             - x86-musl-cross-make
 
-      # t480 is based on 25.09 coreboot release, not sharing any buildstack from now, depend on muscl-cross cache
-      - build_and_persist:
+      # coreboot-25.09
+      - build_and_persist_save_cache_coreboot_x86:
           name: EOL_t480-hotp-maximized
           target: EOL_t480-hotp-maximized
           subcommand: ""
+          coreboot_dir: coreboot-25.09
           requires:
             - x86-musl-cross-make
 
-      # coreboot talos_2
-      - build_and_persist:
-          name: UNTESTED_talos-2
-          arch: ppc64
-          target: UNTESTED_talos-2
-          subcommand: ""
-          requires:
-            - ppc64-musl-cross-make
-
-      # coreboot 4.11
-      - build_and_persist:
+      # coreboot-4.11
+      - build_and_persist_save_cache_coreboot_x86:
           name: EOL_librem_l1um
           target: EOL_librem_l1um
           subcommand: ""
+          coreboot_dir: coreboot-4.11
           requires:
             - x86-musl-cross-make
 
-      # dasharo_msi: builds its own toolchain, msi_z690a_ddr5 depends on it
-      - build_and_persist:
+      # coreboot-dasharo_msi_z690
+      - build_and_persist_save_cache_coreboot_x86:
           name: UNTESTED_msi_z690a_ddr4
           target: UNTESTED_msi_z690a_ddr4
           subcommand: ""
+          coreboot_dir: coreboot-dasharo_msi_z690
           requires:
             - x86-musl-cross-make
 
-      # Dasharo forks: each fork builds its own toolchain (FSP header incompatibilities)
-      # - novacustom-nv4x_adl: builds dasharo_nv4x toolchain, nitropad-ns50 reuses it
-      # - novacustom-v560tu: builds dasharo_v56 toolchain, v540tu reuses it
-      # - UNTESTED_msi_z690a_ddr4: builds dasharo_msi_z690 toolchain, msi_z690a_ddr5 reuses it
-      # - UNTESTED_msi_z790p_ddr4: builds dasharo_msi_z790 toolchain, msi_z790p_ddr5 reuses it
-
-      # x86: combines x86 workspaces and saves x86 cache
-      # Workspaces combined: x86-musl-cross-make, novacustom-nv4x_adl, librem_14,
-      #                      EOL_t480-hotp-maximized, EOL_librem_l1um, UNTESTED_msi_z690a_ddr4
-      - save_cache_x86:
+      # coreboot-dasharo_msi_z790 — last fork seed, also saves full modules cache
+      - build_and_persist_save_cache_coreboot_x86:
+          name: UNTESTED_msi_z790p_ddr4
+          target: UNTESTED_msi_z790p_ddr4
+          subcommand: ""
+          coreboot_dir: coreboot-dasharo_msi_z790
+          save_full_modules_cache: true
           requires:
             - x86-musl-cross-make
-            - novacustom-nv4x_adl
-            - librem_14
-            - EOL_t480-hotp-maximized
-            - EOL_librem_l1um
-            - UNTESTED_msi_z690a_ddr4
-
-      # ppc64: combines ppc64 workspaces and saves ppc64 cache
-      # Workspaces combined: ppc64-musl-cross-make, UNTESTED_talos-2
-      - save_cache_ppc64:
-          requires:
-            - UNTESTED_talos-2
 
       # Those onboarding new boards should add their entries below.
       # coreboot 25.09 boards
@@ -625,14 +683,6 @@ workflows:
           requires:
             - novacustom-nv4x_adl
 
-      # NovaCustom v56: builds dasharo_v56 toolchain
-      - build:
-          name: novacustom-v560tu
-          target: novacustom-v560tu
-          subcommand: ""
-          requires:
-            - x86-musl-cross-make
-
       # NovaCustom v56: shares dasharo_v56 toolchain with novacustom-v560tu
       - build:
           name: novacustom-v540tu
@@ -649,14 +699,6 @@ workflows:
           requires:
             - UNTESTED_msi_z690a_ddr4
 
-      # Dasharo msi_z790: builds dasharo_msi_z790 toolchain
-      - build:
-          name: UNTESTED_msi_z790p_ddr4
-          target: UNTESTED_msi_z790p_ddr4
-          subcommand: ""
-          requires:
-            - x86-musl-cross-make
-
       # Dasharo msi_z790: shares dasharo_msi_z790 toolchain with UNTESTED_msi_z790p_ddr4
       - build:
           name: msi_z790p_ddr5