release-recover-cli.yml

name: Release Recover (moltnet-cli)

# Recovers a stuck `apps/moltnet-cli` release cycle. Handles two scenarios:
#
#   A) goreleaser-failure: cli was released in the same cycle as one or
#      more libs it depends on. release-please bumped the lib versions in
#      the manifest but did NOT sync `apps/moltnet-cli/go.mod`, so
#      goreleaser built against a stale pin. Needs manifest revert +
#      CHANGELOG drop + go.mod sync.
#
#   B) downstream-publish-failure: goreleaser succeeded but one of the
#      npm publish jobs (or a sibling tag creation) failed transiently.
#      The cli tag + release exist as draft and/or the 7 linked-versions
#      sibling tags are half-created, blocking the next release-please
#      run. Needs manifest revert + CHANGELOG drop + draft/tag cleanup.
#      No go.mod sync — no libs were bumped in this cycle.
#
# The workflow auto-detects which mode applies by diffing the cli's
# direct libs/* dependencies in `apps/moltnet-cli/go.mod` against the
# manifest before/after versions.
#
# Usage: trigger this workflow with the number of the merged release-please
# PR that caused the failure (the "chore: release main" PR). Everything
# else — previous/stuck cli versions, which libs to re-pin — is inferred by
# diffing `.release-please-manifest.json` between the PR's merge commit and
# its first parent, then filtering against the cli's actual `libs/` deps in
# `apps/moltnet-cli/go.mod`.
#
# The recipe:
#   1. Resolve merge commit + parent from the PR number
#   2. Diff .release-please-manifest.json between parent and merge commit
#   3. Extract previous + stuck cli versions; collect bumped libs that the
#      cli depends on (auto-discovered from apps/moltnet-cli/go.mod)
#   4. Revert cli version in the manifest to the previous released version
#      for every member of the linked cli group (apps/moltnet-cli,
#      packages/cli, and the 6 packages/cli/npm/* platform packages) so the
#      manifest stays consistent with the linked-versions invariant
#   5. Drop the premature `## [x.y.z]` section from cli CHANGELOG
#   6. `GOWORK=off go get <lib>@<version>` for each bumped lib + go mod tidy
#   7. Commit on a branch, open PR (squash-merge it manually)
#   8. Delete the stuck draft `cli-vX.Y.Z` GitHub release
#
# After merging the PR, release-please will re-propose the cli bump with a
# synced go.mod on main, and the next release-cli run will succeed.

on:
  workflow_dispatch:
    inputs:
      release-pr:
        description: 'Number of the merged release-please PR that caused the failure (e.g. 766)'
        required: true
        type: string
      delete-draft-release:
        description: 'Delete the stuck draft GitHub release cli-v<stuck>'
        required: false
        type: boolean
        default: true

permissions:
  contents: write
  pull-requests: write

jobs:
  recover:
    name: Recover cli release cycle
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          token: ${{ secrets.GITHUB_TOKEN }}

      - uses: actions/setup-go@v6
        with:
          go-version-file: apps/moltnet-cli/go.mod
          cache: false

      - name: Configure git
        run: |
          git config user.name 'github-actions[bot]'
          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'

      - name: Infer recovery parameters from release PR
        id: infer
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ inputs.release-pr }}
        run: |
          set -euo pipefail

          # 1. Fetch PR metadata. Must be merged.
          pr_json=$(gh pr view "$PR_NUMBER" --json state,mergeCommit,title,url)
          state=$(echo "$pr_json" | jq -r '.state')
          merge_sha=$(echo "$pr_json" | jq -r '.mergeCommit.oid // empty')
          pr_title=$(echo "$pr_json" | jq -r '.title')
          pr_url=$(echo "$pr_json" | jq -r '.url')

          if [ "$state" != 'MERGED' ]; then
            echo "ERROR: PR #$PR_NUMBER is $state, expected MERGED" >&2
            exit 1
          fi
          if [ -z "$merge_sha" ]; then
            echo "ERROR: PR #$PR_NUMBER has no merge commit" >&2
            exit 1
          fi

          echo "PR:          #$PR_NUMBER ($pr_url)"
          echo "Title:       $pr_title"
          echo "Merge SHA:   $merge_sha"

          # 2. Parent commit = manifest state before the release PR merged.
          #    We use ^1 (first parent) rather than baseRefOid because the
          #    base branch may have advanced since the merge.
          parent_sha=$(git rev-parse "${merge_sha}^1")
          echo "Parent SHA:  $parent_sha"

          # 3. Read manifest at both commits.
          manifest_path='.release-please-manifest.json'
          git show "${parent_sha}:${manifest_path}" > /tmp/manifest.before.json
          git show "${merge_sha}:${manifest_path}" > /tmp/manifest.after.json

          previous_cli=$(jq -r '."apps/moltnet-cli"' /tmp/manifest.before.json)
          stuck_cli=$(jq -r '."apps/moltnet-cli"' /tmp/manifest.after.json)

          if [ "$previous_cli" = 'null' ] || [ "$stuck_cli" = 'null' ]; then
            echo "ERROR: apps/moltnet-cli missing from manifest at one of the commits" >&2
            exit 1
          fi
          if [ "$previous_cli" = "$stuck_cli" ]; then
            echo "ERROR: apps/moltnet-cli version unchanged in PR #$PR_NUMBER ($previous_cli)" >&2
            echo "       This PR did not bump the cli — nothing to recover." >&2
            exit 1
          fi

          echo "Previous cli: $previous_cli"
          echo "Stuck cli:    $stuck_cli"

          # 4. Auto-discover cli's direct libs/ dependencies from go.mod.
          #    We only look at the first `require (...)` block (direct deps);
          #    indirect deps live in a second block marked `// indirect`.
          cli_libs=$(awk '
            /^require \(/ { in_block=1; next }
            in_block && /^\)/ { in_block=0; next }
            in_block && /github\.com\/getlarge\/themoltnet\/libs\// && !/\/\/ indirect/ {
              # Line format: "\tgithub.com/getlarge/themoltnet/libs/<name> v<ver>"
              sub(/^[[:space:]]+/, "", $0)
              split($0, parts, " ")
              sub(/^github\.com\/getlarge\/themoltnet\/libs\//, "", parts[1])
              print parts[1]
            }
          ' apps/moltnet-cli/go.mod)

          # It is legitimate for cli_libs to be empty (cli has no direct
          # libs/* deps) or for none of them to be bumped in this PR. That
          # corresponds to the "downstream-publish-failure" recovery path:
          # goreleaser already succeeded but some npm publish job failed,
          # so there's no go.mod to sync — we only need to revert the
          # manifest + CHANGELOG + drop draft/tags so release-please can
          # re-propose the same cli bump.
          pairs=()
          if [ -n "$cli_libs" ]; then
            echo 'Cli libs/* deps:'
            echo "$cli_libs" | sed 's/^/  - /'

            # 5. For each cli lib dep, compare before/after manifest versions.
            while IFS= read -r lib; do
              [ -n "$lib" ] || continue
              before=$(jq -r --arg k "libs/${lib}" '.[$k] // empty' /tmp/manifest.before.json)
              after=$(jq -r --arg k "libs/${lib}" '.[$k] // empty' /tmp/manifest.after.json)
              if [ -z "$after" ]; then
                echo "  libs/${lib}: not tracked in manifest (skipping)"
                continue
              fi
              if [ "$before" = "$after" ]; then
                echo "  libs/${lib}: unchanged (${after})"
                continue
              fi
              echo "  libs/${lib}: ${before:-<none>} -> ${after}"
              pairs+=("${lib}=${after}")
            done <<< "$cli_libs"
          else
            echo 'No direct libs/* dependencies in apps/moltnet-cli/go.mod'
          fi

          if [ ${#pairs[@]} -eq 0 ]; then
            echo 'No cli lib dependencies were bumped in this PR.'
            echo 'Recovery mode: downstream-publish-failure (manifest + changelog only, no go.mod sync).'
            bumped_libs=''
          else
            bumped_libs=$(IFS=,; echo "${pairs[*]}")
            echo "Recovery mode: goreleaser-failure (go.mod sync required)."
            echo "Bumped libs: $bumped_libs"
          fi

          # Export outputs for downstream steps.
          {
            echo "previous-cli-version=${previous_cli}"
            echo "stuck-cli-version=${stuck_cli}"
            echo "bumped-libs=${bumped_libs}"
            echo "pr-url=${pr_url}"
          } >> "$GITHUB_OUTPUT"

      # Safety guard: confirm the cli-v<stuck> release is actually stuck
      # before we revert anything. Previously this was covered implicitly
      # by the "at least one lib must be bumped" check, but that check
      # missed the downstream-publish-failure path. Now we check the
      # release/tag/npm state directly.
      #
      # A release is stuck iff ANY of:
      #   a) GitHub release cli-v<stuck> is a draft (goreleaser didn't
      #      finalize it, or the workflow died before the release job
      #      flipped it).
      #   b) GitHub release is published but one or more of the 7
      #      linked-versions sibling tags (cli-wrapper-v<x>,
      #      cli-<platform>-v<x>) is missing.
      #   c) GitHub release is published but one or more of the 7
      #      @themoltnet/cli* npm versions is missing on the registry.
      #
      # If none hold, recovery would destroy a healthy release — refuse.
      - name: Verify cli release is stuck
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          STUCK_VERSION: ${{ steps.infer.outputs.stuck-cli-version }}
        run: |
          set -euo pipefail
          TAG="cli-v${STUCK_VERSION}"
          reasons=()

          # a) Draft release check.
          release_state='missing'
          if release_json=$(gh release view "$TAG" --json isDraft,isPrerelease 2>/dev/null); then
            if [ "$(echo "$release_json" | jq -r '.isDraft')" = 'true' ]; then
              release_state='draft'
              reasons+=("GitHub release $TAG is a draft (goreleaser did not finalize)")
            else
              release_state='published'
            fi
          else
            reasons+=("GitHub release $TAG does not exist (goreleaser never ran or failed early)")
          fi
          echo "Release state: $release_state"

          # b) Sibling tag completeness check (only meaningful if release exists).
          missing_tags=()
          for COMPONENT in cli-wrapper cli-darwin-arm64 cli-darwin-x64 cli-linux-arm64 cli-linux-x64 cli-win32-arm64 cli-win32-x64; do
            SIB_TAG="${COMPONENT}-v${STUCK_VERSION}"
            if ! gh api "repos/${GITHUB_REPOSITORY}/git/refs/tags/${SIB_TAG}" >/dev/null 2>&1; then
              missing_tags+=("$SIB_TAG")
            fi
          done
          if [ ${#missing_tags[@]} -gt 0 ]; then
            reasons+=("Missing ${#missing_tags[@]} sibling tag(s): ${missing_tags[*]}")
          fi

          # c) npm publish completeness check. Registry lookup is public
          #    and read-only, so no auth is needed. `npm view <pkg>@<ver>`
          #    exits 0 if the version exists and prints E404 to stderr
          #    otherwise.
          missing_npm=()
          for PKG in \
            '@themoltnet/cli' \
            '@themoltnet/cli-darwin-arm64' \
            '@themoltnet/cli-darwin-x64' \
            '@themoltnet/cli-linux-arm64' \
            '@themoltnet/cli-linux-x64' \
            '@themoltnet/cli-win32-arm64' \
            '@themoltnet/cli-win32-x64'; do
            if ! npm view "${PKG}@${STUCK_VERSION}" version >/dev/null 2>&1; then
              missing_npm+=("${PKG}@${STUCK_VERSION}")
            fi
          done
          if [ ${#missing_npm[@]} -gt 0 ]; then
            reasons+=("Missing ${#missing_npm[@]} npm publish(es): ${missing_npm[*]}")
          fi

          if [ ${#reasons[@]} -eq 0 ]; then
            echo "ERROR: cli-v${STUCK_VERSION} looks healthy — refusing to recover." >&2
            echo "       Release is published, all 7 sibling tags exist, and all 7" >&2
            echo "       npm packages are on the registry. If you believe this PR is" >&2
            echo "       genuinely stuck, investigate manually before re-running." >&2
            exit 1
          fi

          echo 'Stuckness evidence:'
          for r in "${reasons[@]}"; do
            echo "  - $r"
          done

      - name: Create recovery branch
        id: branch
        env:
          STUCK_VERSION: ${{ steps.infer.outputs.stuck-cli-version }}
        run: |
          BRANCH="chore/recover-cli-v${STUCK_VERSION}"
          git checkout -b "$BRANCH"
          echo "name=$BRANCH" >> "$GITHUB_OUTPUT"

      - name: Revert manifest cli version
        env:
          PREVIOUS_VERSION: ${{ steps.infer.outputs.previous-cli-version }}
        run: |
          # The cli is part of an 8-component linked-versions group
          # (apps/moltnet-cli + packages/cli + 6 platform packages). All
          # members must move in lockstep, so the revert has to touch every
          # path — otherwise the manifest contradicts the linked-versions
          # invariant and release-please gets confused on the next run.
          jq --arg v "$PREVIOUS_VERSION" '
            ."apps/moltnet-cli"                  = $v
            | ."packages/cli"                    = $v
            | ."packages/cli/npm/darwin-arm64"   = $v
            | ."packages/cli/npm/darwin-x64"     = $v
            | ."packages/cli/npm/linux-arm64"    = $v
            | ."packages/cli/npm/linux-x64"      = $v
            | ."packages/cli/npm/win32-arm64"    = $v
            | ."packages/cli/npm/win32-x64"      = $v
          ' .release-please-manifest.json \
            > .release-please-manifest.json.tmp
          mv .release-please-manifest.json.tmp .release-please-manifest.json
          git diff --stat .release-please-manifest.json

      - name: Drop premature CHANGELOG section
        env:
          STUCK_VERSION: ${{ steps.infer.outputs.stuck-cli-version }}
        run: |
          python3 - <<'PY'
          import os, re, sys, pathlib
          p = pathlib.Path('apps/moltnet-cli/CHANGELOG.md')
          text = p.read_text()
          stuck = os.environ['STUCK_VERSION']
          pattern = re.compile(
              r'^## \[?' + re.escape(stuck) + r'\]?.*?(?=^## \[)',
              re.MULTILINE | re.DOTALL,
          )
          new = pattern.sub('', text, count=1)
          if new == text:
              print(f'ERROR: no ## [{stuck}] section found in CHANGELOG', file=sys.stderr)
              sys.exit(1)
          p.write_text(new)
          PY
          git diff --stat apps/moltnet-cli/CHANGELOG.md

      - name: Sync apps/moltnet-cli/go.mod to bumped libs
        if: ${{ steps.infer.outputs.bumped-libs != '' }}
        working-directory: apps/moltnet-cli
        env:
          GOWORK: off
          BUMPED_LIBS: ${{ steps.infer.outputs.bumped-libs }}
        run: |
          IFS=',' read -ra PAIRS <<< "$BUMPED_LIBS"
          for raw in "${PAIRS[@]}"; do
            # Trim whitespace
            pair="$(echo "$raw" | xargs)"
            if [[ "$pair" != *=* ]]; then
              echo "ERROR: invalid bumped-libs entry '$raw' (expected name=version)" >&2
              exit 1
            fi
            name="${pair%%=*}"
            version="${pair##*=}"
            if [ -z "$name" ] || [ -z "$version" ]; then
              echo "ERROR: empty name or version in '$raw'" >&2
              exit 1
            fi
            module="github.com/getlarge/themoltnet/libs/${name}"
            echo "::group::go get ${module}@v${version}"
            go get "${module}@v${version}"
            resolved=$(go list -m "$module" | awk '{print $2}')
            if [ "$resolved" != "v${version}" ]; then
              echo "ERROR: requested v${version} but MVS resolved ${resolved}" >&2
              exit 1
            fi
            echo "::endgroup::"
          done
          go mod tidy
          go build ./...

      - name: Commit and push
        env:
          STUCK_VERSION: ${{ steps.infer.outputs.stuck-cli-version }}
          BRANCH_NAME: ${{ steps.branch.outputs.name }}
        run: |
          git add \
            .release-please-manifest.json \
            apps/moltnet-cli/CHANGELOG.md \
            apps/moltnet-cli/go.mod \
            apps/moltnet-cli/go.sum
          if git diff --cached --quiet; then
            echo 'No changes to commit — nothing to recover.'
            exit 1
          fi
          git commit -m "fix(release): recover cli v${STUCK_VERSION} release cycle"
          git push -u origin "$BRANCH_NAME"

      - name: Open recovery PR
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          STUCK_VERSION: ${{ steps.infer.outputs.stuck-cli-version }}
          PREVIOUS_VERSION: ${{ steps.infer.outputs.previous-cli-version }}
          BUMPED_LIBS: ${{ steps.infer.outputs.bumped-libs }}
          SOURCE_PR_URL: ${{ steps.infer.outputs.pr-url }}
          BRANCH_NAME: ${{ steps.branch.outputs.name }}
        run: |
          if [ -n "$BUMPED_LIBS" ]; then
            GOMOD_LINE="- Syncs \`apps/moltnet-cli/go.mod\` to: \`${BUMPED_LIBS}\`"
          else
            GOMOD_LINE="- No go.mod sync required (no libs bumped in source PR — downstream-publish-failure recovery)"
          fi
          BODY=$(cat <<EOF
          Automated recovery for a stuck \`apps/moltnet-cli\` release cycle.

          **Triggered by:** ${SOURCE_PR_URL}

          **What this PR does:**
          - Reverts all 8 linked cli group entries in \`.release-please-manifest.json\` from \`${STUCK_VERSION}\` back to \`${PREVIOUS_VERSION}\` (\`apps/moltnet-cli\`, \`packages/cli\`, and the 6 platform packages — they must move in lockstep under \`linked-versions\`)
          - Drops the premature \`## [${STUCK_VERSION}]\` section from \`apps/moltnet-cli/CHANGELOG.md\`
          ${GOMOD_LINE}

          **Next steps:**
          1. Squash-merge this PR
          2. release-please will re-propose the cli bump with a synced go.mod
          3. Merging that PR triggers a clean release-cli run

          The recovery recipe is documented in the workflow header at \`.github/workflows/release-recover-cli.yml\`.
          EOF
          )
          gh pr create \
            --base main \
            --head "$BRANCH_NAME" \
            --title "fix(release): recover cli v${STUCK_VERSION} release cycle" \
            --body "$BODY"

      - name: Delete stuck draft release
        if: ${{ inputs.delete-draft-release }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          STUCK_VERSION: ${{ steps.infer.outputs.stuck-cli-version }}
        run: |
          TAG="cli-v${STUCK_VERSION}"
          # Query isDraft so we never delete a published release.
          # Don't suppress stderr — surface 403/429/network errors.
          if ! IS_DRAFT="$(gh release view "$TAG" --json isDraft --jq '.isDraft' 2>/tmp/gh-err)"; then
            if grep -qi 'release not found' /tmp/gh-err; then
              echo "No release $TAG to delete"
              exit 0
            fi
            echo "ERROR: gh release view failed:" >&2
            cat /tmp/gh-err >&2
            exit 1
          fi
          if [ "$IS_DRAFT" = 'true' ]; then
            # Delete the draft release WITHOUT --cleanup-tag. Draft releases
            # don't have a real git tag ref backing them, so `gh release
            # delete --cleanup-tag` tries to DELETE /git/refs/tags/<tag>,
            # gets HTTP 422 "Reference does not exist", and exits non-zero
            # even though the release itself was deleted cleanly.
            gh release delete "$TAG" --yes
            echo "Deleted draft release $TAG"

            # Best-effort tag cleanup: only delete the ref if it actually
            # exists. A stray published tag with the same name would be a
            # separate bug, so we warn but do not hard-fail on delete errors.
            if gh api "repos/${GITHUB_REPOSITORY}/git/refs/tags/${TAG}" >/dev/null 2>&1; then
              if gh api -X DELETE "repos/${GITHUB_REPOSITORY}/git/refs/tags/${TAG}" >/dev/null 2>&1; then
                echo "Deleted leftover tag ref $TAG"
              else
                echo "WARN: failed to delete leftover tag ref $TAG (continuing)" >&2
              fi
            else
              echo "No leftover tag ref $TAG to clean up"
            fi
          else
            echo "Release $TAG exists but is not a draft; skipping deletion"
          fi

      # The linked-versions cli group has 7 sibling tags
      # (cli-wrapper-v<x>, cli-<platform>-v<x>) created by the
      # `tag-cli-siblings` job in release.yml once `release-cli` succeeds.
      # They exist independently of the cli-v<x> GitHub release state, so
      # clean them up with the same toggle as the draft release — if
      # release-cli never succeeded, they won't exist and delete is a no-op.
      - name: Delete stuck sibling tags
        if: ${{ inputs.delete-draft-release }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          STUCK_VERSION: ${{ steps.infer.outputs.stuck-cli-version }}
        run: |
          for COMPONENT in cli-wrapper cli-darwin-arm64 cli-darwin-x64 cli-linux-arm64 cli-linux-x64 cli-win32-arm64 cli-win32-x64; do
            SIB_TAG="${COMPONENT}-v${STUCK_VERSION}"
            if gh api "repos/${GITHUB_REPOSITORY}/git/refs/tags/${SIB_TAG}" >/dev/null 2>&1; then
              if gh api -X DELETE "repos/${GITHUB_REPOSITORY}/git/refs/tags/${SIB_TAG}" >/dev/null 2>&1; then
                echo "  - deleted ${SIB_TAG}"
              else
                echo "  ! WARN: failed to delete ${SIB_TAG} (continuing)" >&2
              fi
            else
              echo "  = ${SIB_TAG} does not exist"
            fi
          done