feat: Release Tracks — floating Docker tags (latest/standard/trailing) promotion engine (#36160)

.github/workflows/cicd_6-release.yml

@@ -139,7 +139,11 @@ jobs:
       tag-identifier: ${{ needs.release-prepare.outputs.release_version }}
       omit-environment-prefix: true
       artifact-run-id: ${{ github.run_id }}
-      latest: ${{ needs.release-prepare.outputs.is_latest == 'true' && github.ref_name == 'main' && github.event.inputs.update_latest == 'true' }}
+      # `latest` is no longer moved here. evergreen-tracks is the single controller
+      # of the floating latest/standard/trailing tags; the promote-latest job below
+      # repoints `latest` on-demand once the release images are published (and honors
+      # the `update_latest` opt-out for back-patches of older release lines).
+      latest: false
       deploy-cli: true
       deploy-dev-image: true
       publish-npm-cli: false
@@ -154,6 +158,50 @@ jobs:
       SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
       DEV_REQUEST_TOKEN: ${{ secrets.DEV_REQUEST_TOKEN }}
 
+  # Promote latest - move the floating `latest` Docker tag to this GA release.
+  #
+  # evergreen-tracks (cicd/evergreen-tracks) is the single controller of the
+  # latest/standard/trailing tags. Its daily cron ages standard/trailing, but
+  # `latest` must move the instant a GA ships — so the release pipeline invokes
+  # the same engine on-demand, scoped to the latest track only (--tracks latest).
+  # This replaces the old deploy-docker `latest: true` path (now `latest: false`).
+  #
+  # Applies for any real latest release from main on dotcms/core.
+  promote-latest:
+    name: Promote latest tag
+    # Default success() gate: only runs when release-prepare AND deployment both
+    # succeeded, so the release images are guaranteed to exist before we repoint.
+    needs: [ release-prepare, deployment ]
+    if: >-
+      needs.release-prepare.outputs.is_latest == 'true'
+      && github.ref_name == 'main'
+      && github.repository == 'dotcms/core'
+      && github.event.inputs.update_latest == 'true'
+    runs-on: ubuntu-${{ vars.UBUNTU_RUNNER_VERSION || '24.04' }}
+    permissions:
+      contents: read
+    # Share the registry-mutation lock with the promote/admin workflows so this
+    # on-demand promote can never race the daily cron or an admin run.
+    concurrency:
+      group: evergreen-tracks-registry
+      cancel-in-progress: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - uses: docker/setup-buildx-action@v3
+      - name: Docker login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Move latest to ${{ needs.release-prepare.outputs.release_version }}
+        working-directory: cicd/evergreen-tracks
+        run: |
+          for repo in dotcms/dotcms dotcms/dotcms-dev; do
+            echo "--- promoting latest on ${repo} ---"
+            uv run evergreen-tracks promote --repo "${repo}" --tracks latest --apply
+          done
+
   # Release - release-specific operations (Artifactory, Javadocs, Plugins, SBOM, Labels)
   # Waits for deployment to complete to safely update labels only if both succeed
   release:

.github/workflows/cicd_evergreen-tracks-admin.yml

@@ -0,0 +1,71 @@
+name: evergreen-tracks-admin
+on:
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: 'Docker repo to operate on'
+        required: true
+        default: 'dotcms/dotcms'
+      action:
+        description: 'taint | untaint | hold | release-hold'
+        required: true
+        type: choice
+        options: [taint, untaint, hold, release-hold]
+      version:
+        description: 'GA version (for taint/untaint/hold)'
+        required: false
+        default: ''
+      track:
+        description: 'Track (for hold/release-hold): latest|standard|trailing'
+        required: false
+        default: ''
+      force:
+        description: 'Allow holding onto a tainted version'
+        required: false
+        default: 'false'
+      apply:
+        description: 'Actually mutate tags (false = dry-run)'
+        required: true
+        default: 'false'
+
+# Share the registry-mutation lock with the promote workflows so an admin
+# taint/hold can never overlap a promote run on the same tags.
+concurrency:
+  group: evergreen-tracks-registry
+  cancel-in-progress: false
+
+jobs:
+  admin:
+    runs-on: ubuntu-${{ vars.UBUNTU_RUNNER_VERSION || '24.04' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - uses: docker/setup-buildx-action@v3
+      - name: Docker login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Run admin action
+        working-directory: cicd/evergreen-tracks
+        env:
+          # Scope Docker Hub creds to this step only (the CLI's untaint/release-hold
+          # delete calls read them); other steps/actions don't need them.
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+          REPO: ${{ github.event.inputs.repo }}
+          ACTION: ${{ github.event.inputs.action }}
+          VERSION: ${{ github.event.inputs.version }}
+          TRACK: ${{ github.event.inputs.track }}
+          FORCE: ${{ github.event.inputs.force }}
+          APPLY: ${{ github.event.inputs.apply }}
+        run: |
+          EXTRA=""
+          if [ "$FORCE" = "true" ]; then EXTRA="$EXTRA --force"; fi
+          if [ "$APPLY" = "true" ]; then EXTRA="$EXTRA --apply"; fi
+          uv run evergreen-tracks admin \
+            --repo "$REPO" \
+            --action "$ACTION" \
+            --version "$VERSION" \
+            --track "$TRACK" \
+            $EXTRA

.github/workflows/cicd_evergreen-tracks-promote.yml

@@ -0,0 +1,76 @@
+name: evergreen-tracks-promote
+on:
+  schedule:
+    - cron: '17 6 * * *'   # daily at 06:17 UTC
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: 'Docker repo to operate on'
+        required: true
+        default: 'dotcms/dotcms'
+      apply:
+        description: 'Actually move tags (false = dry-run)'
+        required: true
+        default: 'false'
+      standard_days:
+        description: 'Min age (days) for the standard track'
+        required: false
+        default: '14'
+      trailing_days:
+        description: 'Min age (days) for the trailing track'
+        required: false
+        default: '28'
+
+permissions:
+  contents: read
+
+# Serialize every registry mutation. The daily cron, a manual promote/admin
+# dispatch, and the on-demand latest-promote in the release pipeline all read
+# live registry state and then apply tag moves, so two concurrent runs could act
+# on stale state and overwrite a hold/taint or move a track based on outdated
+# data. A single static group (not keyed by workflow/ref) makes them serialize
+# across all three workflows. For a tag-promotion engine queueing is safer than
+# cancelling, so cancel-in-progress is false: concurrent runs wait their turn
+# rather than aborting mid-promote and leaving tags half-moved.
+concurrency:
+  group: evergreen-tracks-registry
+  cancel-in-progress: false
+
+jobs:
+  promote:
+    runs-on: ubuntu-${{ vars.UBUNTU_RUNNER_VERSION || '24.04' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - uses: docker/setup-buildx-action@v3
+      - name: Docker login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Promote tracks
+        working-directory: cicd/evergreen-tracks
+        env:
+          # Scheduled runs default to dotcms/dotcms; dispatch can override.
+          REPO: ${{ github.event.inputs.repo || 'dotcms/dotcms' }}
+          STANDARD_DAYS: ${{ github.event.inputs.standard_days || '14' }}
+          TRAILING_DAYS: ${{ github.event.inputs.trailing_days || '28' }}
+          EVENT_NAME: ${{ github.event_name }}
+          DISPATCH_APPLY: ${{ github.event.inputs.apply }}
+        run: |
+          # Scheduled (cron) runs always apply — this is the engine that ages
+          # standard/trailing forward. To pause promotion, disable this workflow
+          # in the Actions tab or `hold` the affected track; there is no separate
+          # apply gate. Manual dispatch defaults to dry-run unless apply=true.
+          APPLY=""
+          if [ "$EVENT_NAME" = "schedule" ]; then
+            APPLY="--apply"
+          elif [ "$EVENT_NAME" = "workflow_dispatch" ] && [ "$DISPATCH_APPLY" = "true" ]; then
+            APPLY="--apply"
+          fi
+          echo "event=$EVENT_NAME repo=$REPO apply=${APPLY:-<dry-run>}"
+          uv run evergreen-tracks promote \
+            --repo "$REPO" \
+            --standard-days "$STANDARD_DAYS" \
+            --trailing-days "$TRAILING_DAYS" \
+            $APPLY

README.md

@@ -53,3 +53,43 @@ For a complete list of requirements, see [this page](http://www.dotcms.com/docs/
 | Forums/Listserv | [via Google Groups](https://groups.google.com/forum/#!forum/dotCMS) |
 | Twitter         | @dotCMS                                                             |
 | Main Site       | [dotCMS.com](https://www.dotcms.com/)                                   |
+
+## Release Tracks
+
+dotCMS GA releases follow CalVer (`YY.0M.0D-NN`, e.g. `26.06.11-01`). On top of that linear
+release stream we publish three **floating Docker tags** — *release tracks* — so you can pick
+how fresh a release each environment receives:
+
+| Tag                       | Age of release    | Use for                                   |
+| ------------------------- | ----------------- | ----------------------------------------- |
+| `dotcms/dotcms:latest`    | newest GA (~days) | tracking the latest release               |
+| `dotcms/dotcms:standard`  | ~2 weeks old      | a short soak before adopting a release    |
+| `dotcms/dotcms:trailing`  | ~4 weeks old      | the most conservative posture             |
+
+Pin the track you want in your deployment manifest, e.g. `image: dotcms/dotcms:standard`, and
+you will roll forward automatically as releases age into that track. Pin an exact version
+(`dotcms/dotcms:26.06.11-01`) instead if you never want automatic movement.
+
+One engine (under [`cicd/evergreen-tracks/`](cicd/evergreen-tracks/)) re-points every track tag
+by image digest, on two triggers: the release pipeline moves `latest` on-demand the moment a GA
+ships, and a daily scheduled job ages `standard`/`trailing` forward. Two design choices are worth
+understanding:
+
+- **Age is measured from the CalVer date in the version string, not from when the image was
+  built or published.** This protects emergency backports: if we cut a patch of an *older*
+  release on short notice, it is built today but logically belongs to the older release line.
+  Using the build/publish date would make that patch look brand new and let it jump onto the
+  `standard`/`trailing` tracks ahead of releases that are genuinely older. Anchoring to the
+  embedded CalVer date keeps every release in its true place on the timeline.
+- **A track never moves backward automatically, and a release can be "tainted".** If a bad
+  release is found, it is tainted so it will not advance onto tracks it has not yet reached —
+  a known-bad build can never propagate from `latest` down to `standard`/`trailing`. A track
+  can also be **held** (frozen) at a specific version for incident response. Both controls live
+  as marker tags in the registry; there is no separate datastore.
+
+> **GitOps / Argo note:** because these tags float (the same tag is re-pointed to newer
+> digests over time), a tag reference alone will not trigger a redeploy. Use Argo CD Image
+> Updater or a periodic rollout refresh to pick up track movements.
+
+The promotion engine and its scheduled/admin workflows live under
+[`cicd/evergreen-tracks/`](cicd/evergreen-tracks/).

cicd/evergreen-tracks/README.md

@@ -0,0 +1,16 @@
+# evergreen-tracks
+
+Advances floating Docker track tags (`latest`, `standard`, `trailing`) across the dotCMS
+GA CalVer release stream by release age. State lives entirely in registry tags (the track
+tags plus `<version>_tainted` / `<track>_hold` markers).
+
+## Run locally (dry-run is the default)
+
+    uv run evergreen-tracks promote --repo dotcms/dotcms-test
+    uv run evergreen-tracks admin --repo dotcms/dotcms-test --action taint --version 26.03.12-01
+
+Pass `--apply` to actually move tags. Without it, the command prints the plan and exits.
+
+## Test
+
+    uv run pytest

cicd/evergreen-tracks/pyproject.toml

@@ -0,0 +1,27 @@
+[project]
+name = "evergreen-tracks"
+version = "0.1.0"
+description = "Floating Docker track tags (latest/standard/trailing) for dotCMS releases"
+requires-python = ">=3.12"
+dependencies = [
+    "requests>=2.32",
+]
+
+[project.scripts]
+evergreen-tracks = "evergreen_tracks.cli:main"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "responses>=0.25",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evergreen_tracks"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

cicd/evergreen-tracks/src/evergreen_tracks/__init__.py

@@ -0,0 +1,3 @@
+"""Evergreen release tracks: floating Docker tags advanced by release age."""
+
+__all__ = []

cicd/evergreen-tracks/src/evergreen_tracks/calver.py

@@ -0,0 +1,42 @@
+"""CalVer release parsing, age, and ordering. No I/O."""
+from __future__ import annotations
+
+import datetime as dt
+import re
+from dataclasses import dataclass
+
+# GA tags only: YY.0M.0D-NN  (e.g. 26.06.11-01). Excludes SNAPSHOT, RC, _lts*, _javaNN, markers.
+GA_RE = re.compile(r"^(\d{2})\.(\d{2})\.(\d{2})-(\d{1,2})$")
+
+
+@dataclass(frozen=True)
+class Release:
+    version: str        # original tag, e.g. "26.06.11-01"
+    date: dt.date       # parsed from YY.0M.0D
+    build: int          # NN
+
+    @property
+    def sort_key(self) -> tuple[dt.date, int]:
+        return (self.date, self.build)
+
+
+def parse_release(tag: str) -> Release | None:
+    """Return a Release for a GA tag, or None for anything else."""
+    m = GA_RE.match(tag)
+    if not m:
+        return None
+    yy, mm, dd, nn = m.groups()
+    try:
+        date = dt.date(2000 + int(yy), int(mm), int(dd))
+    except ValueError:
+        return None
+    return Release(version=tag, date=date, build=int(nn))
+
+
+def age_days(release: Release, today: dt.date) -> int:
+    return (today - release.date).days
+
+
+def newest(releases: list[Release]) -> Release | None:
+    """Newest by (date, build); None if the list is empty."""
+    return max(releases, key=lambda r: r.sort_key, default=None)