deploy to docker hub in series to address concurrency

maureeungaro · maureeungaro · commit 5bdab7b63d22 · 2026-06-18T21:57:48.000-04:00
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -205,11 +205,6 @@ jobs:
       matrix: ${{ fromJSON(needs.discover.outputs.matrix_manifest) }}
     env:
       TAG4: ${{ format('{0}-{1}-{2}', matrix.gemc_tag, matrix.image, matrix.image_tag) }}
-      # Docker Hub mirror target. Steps are skipped automatically when the
-      # DOCKERHUB_USERNAME secret is not configured (e.g. on forks).
-      # Override the namespace with the DOCKERHUB_REPO repo variable.
-      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-      DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO || 'docker.io/maureeungaro/gemc' }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v6
@@ -223,14 +218,6 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Log in to Docker Hub
-        if: ${{ env.DOCKERHUB_USERNAME != '' }}
-        uses: docker/login-action@v4
-        with:
-          registry: docker.io
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Set up Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -256,17 +243,6 @@ jobs:
             docker buildx imagetools create -t "$BASE_TAG" "$AMD_TAG"
           fi
 
-      - name: Mirror manifest to Docker Hub
-        if: ${{ env.DOCKERHUB_USERNAME != '' }}
-        shell: bash
-        run: |
-          # Copy the already-built multi-arch manifest from GHCR to Docker Hub
-          # registry-to-registry (no pull, no rebuild).
-          SRC="${{ needs.discover.outputs.image }}:${{ env.TAG4 }}"
-          DST="${DOCKERHUB_REPO}:${{ env.TAG4 }}"
-          echo "Mirroring $SRC -> $DST"
-          docker buildx imagetools create -t "$DST" "$SRC"
-
       - name: Download logs artifacts (all arches)
         if: ${{ always() }}
         uses: actions/download-artifact@v7
@@ -298,12 +274,85 @@ jobs:
           path: ${{ env.MANIFEST_SUMMARY_FILE }}
           if-no-files-found: warn
 
-  # Push the Docker Hub overview README once per deploy. Skipped automatically
-  # when the DOCKERHUB_USERNAME secret is absent (e.g. on forks).
+  # Mirror the GHCR multi-arch manifests to Docker Hub. This runs as a single
+  # serial job (NOT inside the manifest matrix): all OS families target the same
+  # Docker Hub repo, and concurrent blob uploads to one repo trip Docker Hub's
+  # rate limits, so tags are copied one at a time with retries. The copy is
+  # registry-to-registry (no pull, no rebuild). Skipped when the
+  # DOCKERHUB_USERNAME secret is absent (e.g. on forks).
+  dockerhub_mirror:
+    name: Mirror images to Docker Hub
+    needs: [ discover, manifest ]
+    if: ${{ always() && needs.manifest.result == 'success' }}
+    runs-on: ubuntu-latest
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO || 'docker.io/maureeungaro/gemc' }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      - name: Log in to GHCR
+        if: ${{ env.DOCKERHUB_USERNAME != '' }}
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Log in to Docker Hub
+        if: ${{ env.DOCKERHUB_USERNAME != '' }}
+        uses: docker/login-action@v4
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Buildx
+        if: ${{ env.DOCKERHUB_USERNAME != '' }}
+        uses: docker/setup-buildx-action@v4
+
+      - name: Mirror all tags (sequential, with retry)
+        if: ${{ env.DOCKERHUB_USERNAME != '' }}
+        shell: bash
+        env:
+          SRC_IMAGE: ${{ needs.discover.outputs.image }}
+        run: |
+          set -uo pipefail
+          source ci/tags_config.sh
+
+          copy_with_retry() {
+            local src=$1 dst=$2 n=0 max=5
+            until docker buildx imagetools create -t "$dst" "$src"; do
+              n=$((n + 1))
+              if [ "$n" -ge "$max" ]; then
+                echo "::error::failed to mirror $src -> $dst after $max attempts"
+                return 1
+              fi
+              echo "retry $n/$max for $dst; backing off $((n * 15))s"
+              sleep $((n * 15))
+            done
+            echo "mirrored $src -> $dst"
+          }
+
+          rc=0
+          for gemcv in $(get_gemc_tags); do
+            for pair in "${OS_VERSIONS[@]}"; do
+              os="${pair%%=*}"; ver="${pair#*=}"
+              tag="${gemcv}-${os}-${ver}"
+              copy_with_retry "${SRC_IMAGE}:${tag}" "${DOCKERHUB_REPO}:${tag}" || rc=1
+            done
+          done
+          exit "$rc"
+
+  # Push the Docker Hub overview README once per deploy, after the images exist.
+  # Skipped automatically when the DOCKERHUB_USERNAME secret is absent.
   dockerhub_readme:
     name: Update Docker Hub description
-    needs: manifest
-    if: ${{ always() && needs.manifest.result == 'success' }}
+    needs: dockerhub_mirror
+    if: ${{ always() && needs.dockerhub_mirror.result == 'success' }}
     runs-on: ubuntu-latest
     env:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -337,7 +386,7 @@ jobs:
         github.event.workflow_run.conclusion == 'success' &&
         github.event.workflow_run.event == 'push'
       }}
-    needs: [ build_arch, manifest ]
+    needs: [ build_arch, manifest, dockerhub_mirror, dockerhub_readme ]
     runs-on: ubuntu-latest
     steps:
       - name: Fail if any required job failed
diff --git a/gemc/gsystem/gsystem_options.cc b/gemc/gsystem/gsystem_options.cc
@@ -92,7 +92,8 @@ GOptions defineOptions() {
 		{"name", goptions::NODFLT, "system name (mandatory). For ascii factories, it may include the path to the file"},
 		{"factory", GSYSTEMSQLITETFACTORYLABEL, "factory name."},
 		{"variation", "default", "geometry variation"},
-		{"annotations", UNINITIALIZEDSTRINGQUANTITY, "optional system annotations. Examples: \"mats_only\" "}
+		{"annotations", UNINITIALIZEDSTRINGQUANTITY, "optional system annotations. Examples: \"mats_only\" "},
+		{"digitization", UNINITIALIZEDSTRINGQUANTITY, "optional digitization plugin name when it differs from the system name (shared plugin, e.g. \"ecal\" for the EC and PCAL systems)"}
 	};
 	goptions.defineOption(GSYSTEM_LOGGER, "defines the group of volumes in a system", gsystem, help);
 
diff --git a/gemc_options.cc b/gemc_options.cc
@@ -190,8 +190,18 @@ namespace gemc {
                     if (!entry["name"]) continue;
                     std::string name;
                     try { name = entry["name"].as<std::string>(); } catch (...) { continue; }
-                    if (name.empty() || !seen_names.insert(name).second) continue;
-                    probe_plugin(name + ".gplugin");
+                    if (name.empty()) continue;
+                    if (seen_names.insert(name).second) probe_plugin(name + ".gplugin");
+
+                    // A gsystem may be digitized by a plugin whose name differs from the system
+                    // name (e.g. the EC and PCAL systems share the "ecal" plugin). Probe that
+                    // plugin too so its options and verbosity domain are registered before the
+                    // factory is instantiated.
+                    if (entry["digitization"]) {
+                        std::string dig;
+                        try { dig = entry["digitization"].as<std::string>(); } catch (...) { dig.clear(); }
+                        if (!dig.empty() && seen_names.insert(dig).second) probe_plugin(dig + ".gplugin");
+                    }
                 }
             }
 
diff --git a/meson/g4_pkgconfig.py b/meson/g4_pkgconfig.py
@@ -7,10 +7,11 @@
   - geant4_core.pc   : libs with graphical/GUI/vis/X11/OpenGL/Qt removed
 
 Usage:
-    ./g4_pkgconfig.py <install-prefix>
+    ./g4_pkgconfig.py <install-prefix> [geant4-config]
 
 Example:
     ./g4_pkgconfig.py $GEMC
+    ./g4_pkgconfig.py $GEMC $G4INSTALL/bin/geant4-config
 """
 import subprocess
 import sys
@@ -21,7 +22,7 @@
 print(f"[debug] Python executable: {sys.executable}")
 
 
-# ────────────────────────── helpers ──────────────────────────
+# Helpers
 def run_config(command: str, option: str) -> str:
 	try:
 		result = subprocess.run([command, option], capture_output=True, text=True, check=True)
@@ -172,22 +173,22 @@ def generate_pkgconfig(install_prefix: Path,
 	print(f"Generated {pc_path}")
 
 
-# ────────────────────────── main ──────────────────────────
+# Main
 if __name__ == "__main__":
-	if len(sys.argv) != 2:
-		sys.exit(f"Usage: {sys.argv[0]} <install-prefix>")
+	if len(sys.argv) not in (2, 3):
+		sys.exit(f"Usage: {sys.argv[0]} <install-prefix> [geant4-config]")
 
 	install_dir = Path(sys.argv[1]).expanduser().resolve()
+	config_cmd = sys.argv[2] if len(sys.argv) == 3 else "geant4-config"
 	if not install_dir.exists():
 		print(f"Creating installation directory {install_dir}")
 		install_dir.mkdir(parents=True, exist_ok=True)
 
-	generate_pkgconfig(install_dir, "geant4-config",
+	generate_pkgconfig(install_dir, config_cmd,
 					   "geant4.pc", "Geant4", "Geant4 Simulation Toolkit")
 
-	generate_pkgconfig(install_dir, "geant4-config",
+	generate_pkgconfig(install_dir, config_cmd,
 					   "geant4_core.pc", "Geant4 Core",
 					   "Geant4 Simulation Toolkit (core, no graphical/GUI libs)",
 					   cflags_filter=filter_graphical_cflags,
 					   libs_filter=filter_graphical_libs)
-
diff --git a/meson/meson.build b/meson/meson.build
@@ -36,7 +36,11 @@ expat_dep = dependency('expat', required: true)
 zlib_dep = dependency('zlib', static : false, required : true)  # Use shared libz.so instead of libz.az
 
 # pkg-config --cflags --libs clhep
-clhep_deps = dependency('clhep',  version : '>=2.4.7.1',  include_type : 'system') # treat as system include paths - no warnings
+clhep_deps = dependency(
+    'clhep',
+    version      : '>=2.4.7.1',
+    include_type : 'system',
+)
 
 # feature: feature toggle (uncomment options() above)
 root_opt = get_option('root')
@@ -89,19 +93,44 @@ if (not root_dep.found()) and (not root_opt.disabled())
     endif
 endif
 
-# geant4.pc if not found
+# Generate Geant4 pkg-config files from the active module so downstream builds
+# do not pick stale global pc files from unrelated pkg-config directories.
 fs = import('fs')
 gemc_prefix = get_option('prefix')
 geant4_pc_path = gemc_prefix / 'lib/pkgconfig'
-geant4_core_pc = geant4_pc_path / 'geant4_core.pc'
-geant4_pc = geant4_pc_path / 'geant4.pc'
 g4_pkgconfig_script = meson.project_source_root() / 'meson/g4_pkgconfig.py'
-if not fs.exists(geant4_pc) or not fs.exists(geant4_core_pc)
-    message('geant4.pc or geant4_core.pc  not found in ' + geant4_pc_path + ', running install script to install it in ' + gemc_prefix)
-    run_command(g4_pkgconfig_script, gemc_prefix, check : true)
+
+g4install = run_command('sh', '-c', 'printf %s "$G4INSTALL"', check : false).stdout().strip()
+geant4_config = find_program('geant4-config', required : false)
+if g4install != ''
+    geant4_module_config = find_program(g4install / 'bin' / 'geant4-config', required : false)
+    if geant4_module_config.found()
+        geant4_config = geant4_module_config
+    endif
+endif
+
+if not geant4_config.found()
+    error('geant4-config was not found. Load a Geant4 module before configuring GEMC.')
 endif
-geant4_dep = dependency('geant4', version : '>=11.3.2', method : 'pkg-config', include_type : 'system') # treat as system include paths - no warnings
+
+geant4_config_version = run_command(geant4_config, '--version', check : true).stdout().strip()
+message('Generating Geant4 pkg-config files in ' + geant4_pc_path + ' from ' + geant4_config.full_path())
+run_command(g4_pkgconfig_script, gemc_prefix, geant4_config.full_path(), check : true)
+
+geant4_dep = dependency(
+    'geant4',
+    version      : '>=11.3.2',
+    method       : 'pkg-config',
+    include_type : 'system',
+)
 geant4_core_dep = dependency('geant4_core', version : '>=11.3.2', method : 'pkg-config')
+if geant4_core_dep.version() != geant4_config_version
+    error(
+        'pkg-config resolved geant4_core ' + geant4_core_dep.version() +
+        ', but active geant4-config reports ' + geant4_config_version +
+        '. Put ' + geant4_pc_path + ' before stale pkg-config directories and reconfigure.',
+    )
+endif
 geant4_deps = [geant4_dep]
 geant4_core_deps = [geant4_core_dep]
 if sys == 'linux'
@@ -135,7 +164,8 @@ cmake_opts.add_cmake_defines({
                                  'BUILD_STATIC_LIBS' : true,
                                  'ZLIB_USE_STATIC_LIBS' : false, # prefer shared zlib
                                  'CMAKE_POSITION_INDEPENDENT_CODE' : true,
-                                 'CMAKE_C_STANDARD' : 17, # avoid C23 warnings in bundled zlib if you ever enable it
+                                 # Avoid C23 warnings in bundled zlib if you ever enable it.
+                                 'CMAKE_C_STANDARD' : 17,
                                  'CMAKE_POLICY_VERSION_MINIMUM' : '4.0',
                              })
 
