From adab06d0722b9bbef8e80e3c21e006c6dbec2d35 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 20 May 2026 14:30:54 -0400 Subject: [PATCH 1/4] fix(xtest): remove standalone otdfctl source path; route go through platform monorepo The archived opentdf/otdfctl repo is no longer source-cloned. All Go heads and post-v0.31.0 release tags resolve against opentdf/platform with the otdfctl/ tag infix; bare semver inputs auto-prepend otdfctl/. Pre-v0.31.0 tags fall back to the Go module proxy for artifact install only. Builds of the platform-embedded otdfctl now run with GOWORK pointing at the platform checkout's go.work (auto-detected by the Makefile from each src/{tag} symlink target), so protocol/go, sdk, and lib/* resolve to the local platform sources instead of the otdfctl go.mod's pinned releases. This fixes the failing platform PR scenario where a single PR adds proto symbols and consumes them from otdfctl/. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/xtest.yml | 105 ++----- otdf-sdk-mgr/README.md | 4 +- otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py | 71 ++++- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py | 13 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 57 ++-- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 32 +- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 18 +- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 280 ++++++++++-------- otdf-sdk-mgr/tests/test_resolve.py | 93 +++++- xtest/sdk/go/Makefile | 10 +- xtest/setup-cli-tool/action.yaml | 41 ++- 11 files changed, 431 insertions(+), 293 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index ed89c4b0b..08dfff506 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -28,11 +28,6 @@ on: type: string default: all description: "SDK to focus on (go, js, java, all)" - otdfctl-source: - required: false - type: string - default: auto - description: "otdfctl source: 'auto' (standalone for releases, detect platform for head builds), 'standalone', or 'platform'" workflow_call: inputs: platform-ref: @@ -55,10 +50,6 @@ on: required: false type: string default: all - otdfctl-source: - required: false - type: string - default: auto schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -88,14 +79,6 @@ jobs: JS_REF: "${{ inputs.js-ref }}" OTDFCTL_REF: "${{ inputs.otdfctl-ref }}" JAVA_REF: "${{ inputs.java-ref }}" - # When explicitly set to 'platform', tells the Go resolver to resolve - # against opentdf/platform tags instead of the standalone otdfctl repo. - # In 'auto' mode, releases resolve from standalone; the detect-otdfctl - # step later probes the platform checkout for an embedded otdfctl - # directory, and setup-cli-tool acts on this only for versions that need - # a source checkout (head or artifact-install failure), reusing the - # platform checkout only when the resolved SHA matches. - OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input if: ${{ inputs.focus-sdk != '' }} @@ -322,41 +305,29 @@ jobs: with: node-version: "22.x" - ######## DETECT PLATFORM-EMBEDDED OTDFCTL ############# - - name: Detect platform-embedded otdfctl - id: detect-otdfctl - run: | - if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then - if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then - echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" - echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" - echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" - platform_sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD) || { - echo "::error::Failed to get SHA from platform checkout at $PLATFORM_DIR" - exit 1 - } - echo "otdfctl-sha=$platform_sha" >> "$GITHUB_OUTPUT" - else - echo "otdfctl not found in platform checkout; using standalone repo" - echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" - fi - elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then - if [ -z "$PLATFORM_DIR" ] || [ ! -d "$PLATFORM_DIR/otdfctl" ] || [ ! -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then - echo "::error::otdfctl-source=platform requested but ${PLATFORM_DIR:-}/otdfctl does not exist or lacks go.mod" - exit 1 - fi - echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" - echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" - platform_sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD) || { + ######## CAPTURE PLATFORM-EMBEDDED OTDFCTL LOCATION ############# + # If this matrix's platform checkout contains otdfctl/ (post-v0.31.0 + # tag or main), the setup-cli-tool action reuses it via symlink when an + # otdfctl version resolves to the same SHA. For older platform tags + # without otdfctl/, both outputs stay empty and setup-cli-tool does a + # fresh opentdf/platform checkout for each go head version. + - name: Capture platform otdfctl location + id: platform-otdfctl + run: |- + if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD) || { echo "::error::Failed to get SHA from platform checkout at $PLATFORM_DIR" exit 1 } - echo "otdfctl-sha=$platform_sha" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "Reusable platform-embedded otdfctl at $PLATFORM_DIR/otdfctl ($sha)" else - echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + echo "Platform checkout at $PLATFORM_DIR has no otdfctl/; each go head will get a fresh platform checkout." + echo "dir=" >> "$GITHUB_OUTPUT" + echo "sha=" >> "$GITHUB_OUTPUT" fi env: - OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} ######### CHECKOUT JS CLI ############# @@ -393,8 +364,8 @@ jobs: path: otdftests/xtest/sdk sdk: go version-info: "${{ needs.resolve-versions.outputs.go }}" - platform-otdfctl-dir: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} - platform-otdfctl-sha: ${{ steps.detect-otdfctl.outputs.otdfctl-sha }} + platform-otdfctl-dir: ${{ steps.platform-otdfctl.outputs.dir }} + platform-otdfctl-sha: ${{ steps.platform-otdfctl.outputs.sha }} - name: Cache Go modules if: fromJson(steps.configure-go.outputs.heads)[0] != null @@ -407,38 +378,14 @@ jobs: restore-keys: | go-${{ runner.os }}- - - name: Resolve otdfctl heads - id: resolve-otdfctl-heads - if: fromJson(steps.configure-go.outputs.heads)[0] != null - run: |- - echo "OTDFCTL_HEADS=$OTDFCTL_HEADS" >> "$GITHUB_ENV" - env: - OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }} - - - name: Replace otdfctl go.mod packages, but only at head version of platform - if: >- - steps.detect-otdfctl.outputs.otdfctl-source != 'platform' - && fromJson(steps.configure-go.outputs.heads)[0] != null - && startsWith(matrix.sdk-version, 'go@') - && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) - env: - PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} - run: |- - echo "Replacing go.mod packages (standalone otdfctl)..." - PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}" - OTDFCTL_DIR_ABS="$(pwd)/otdftests/xtest/sdk/go/src/" - echo "PLATFORM_DIR_ABS: $PLATFORM_DIR_ABS" - echo "OTDFCTL_DIR_ABS: $OTDFCTL_DIR_ABS" - for head in $(echo "${OTDFCTL_HEADS}" | jq -r '.[]'); do - echo "Processing head: $head" - cd "${OTDFCTL_DIR_ABS}/$head" - for m in lib/fixtures lib/ocrypto protocol/go sdk; do - go mod edit -replace "github.com/opentdf/platform/$m=${PLATFORM_DIR_ABS}/$m" - done - go mod tidy - done - ######## SETUP THE GO CLI ############# + # otdfctl head builds need the platform's go.work in scope so + # protocol/go, sdk, and lib/* resolve to the platform checkout's local + # sources (otherwise the otdfctl go.mod's pinned release versions are + # used, missing any same-PR additions). The Makefile auto-detects + # GOWORK by looking for a go.work file alongside each src/{tag} symlink + # target — works for both the platform-embedded reuse case and the + # fresh platform-src/{tag} checkout case. - name: Prepare go cli if: fromJson(steps.configure-go.outputs.heads)[0] != null run: |- diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index 1e3fd253f..44a39e027 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -58,13 +58,13 @@ otdf-sdk-mgr java-fixup ## How Release Installs Work -- **Go**: Writes a `.version` file containing `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`); `cli.sh`/`otdfctl.sh` use `go run @` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. +- **Go**: Writes a `.version` file containing `module-path@version`; `cli.sh`/`otdfctl.sh` use `go run @` (no local compilation needed, Go caches the binary). The resolver always targets the opentdf/platform monorepo (`github.com/opentdf/platform/otdfctl`) for v0.31.0 and newer; pre-v0.31.0 tags fall back to the archived standalone repo (`github.com/opentdf/otdfctl`) for artifact install via the Go module proxy. Bare semver inputs like `v0.32.0` are auto-prepended with `otdfctl/` when looking up platform tags. - **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/` - **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar` ## Source Builds -Source builds (`tip` mode) check out source to `sdk/{lang}/src/` and compile via `make` to `sdk/{lang}/dist/`. +Source builds (`tip` mode) check out source to `sdk/{lang}/src/` and compile via `make` to `sdk/{lang}/dist/`. For Go, the platform monorepo is cloned to `sdk/go/platform-src/{ref}/` and `sdk/go/src/{ref}` is a symlink to its `otdfctl/` subdirectory; `make` discovers the platform's top-level `go.work` automatically so `protocol/go`, `sdk`, and `lib/*` resolve to the local checkout. After changes to SDK source, rebuild: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py index 8d2eed71d..b3009ed91 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py @@ -3,6 +3,7 @@ from __future__ import annotations import subprocess +from pathlib import Path from typing import Any from otdf_sdk_mgr.config import SDK_BARE_REPOS, SDK_GIT_URLS, get_sdk_dirs @@ -16,7 +17,18 @@ def _run(cmd: list[str], **kwargs: Any) -> None: def checkout_sdk_branch(language: str, branch: str) -> None: - """Clone bare repo and create/update a worktree for the given branch.""" + """Clone bare repo and create/update a worktree for the given branch. + + For "go", use checkout_go_from_platform instead — the standalone + opentdf/otdfctl repo is archived; otdfctl source builds come from the + platform monorepo. + """ + if language == "go": + raise ValueError( + "checkout_sdk_branch does not support 'go'; use checkout_go_from_platform " + "(otdfctl is now sourced from the opentdf/platform monorepo)." + ) + sdk_dirs = get_sdk_dirs() if language not in sdk_dirs: raise ValueError( @@ -56,3 +68,60 @@ def checkout_sdk_branch(language: str, branch: str) -> None: branch, ] ) + + +def checkout_go_from_platform(ref: str) -> Path: + """Check out the opentdf/platform monorepo and arrange xtest/sdk/go/src/ + as a symlink to the otdfctl subdir of that checkout. + + Returns the platform checkout directory so callers can locate go.work for + GOWORK-based source builds. + """ + go_dir = get_sdk_dirs()["go"] + platform_url = SDK_GIT_URLS["platform"].removesuffix(".git") + platform_src_dir = go_dir / "platform-src" + bare_repo_path = platform_src_dir / "platform.git" + + local_name = ref.replace("/", "--").removeprefix("otdfctl--") + worktree_path = platform_src_dir / local_name + src_link = go_dir / "src" / local_name + + platform_src_dir.mkdir(parents=True, exist_ok=True) + + if not bare_repo_path.exists(): + print(f"Cloning {platform_url} as a bare repository into {bare_repo_path}...") + _run(["git", "clone", "--bare", platform_url, str(bare_repo_path)]) + else: + print(f"Bare repository already exists at {bare_repo_path}. Fetching updates...") + _run(["git", f"--git-dir={bare_repo_path}", "fetch", "--all"]) + + if worktree_path.exists(): + print(f"Worktree for ref '{ref}' already exists at {worktree_path}. Updating...") + _run(["git", "-C", str(worktree_path), "pull", "origin", ref]) + else: + print(f"Setting up worktree for ref '{ref}' at {worktree_path}...") + _run( + [ + "git", + f"--git-dir={bare_repo_path}", + "worktree", + "add", + str(worktree_path), + ref, + ] + ) + + otdfctl_dir = worktree_path / "otdfctl" + if not otdfctl_dir.is_dir(): + raise RuntimeError( + f"Platform checkout at {worktree_path} has no otdfctl/ directory; " + f"ref '{ref}' predates the otdfctl monorepo migration (v0.31.0)." + ) + + src_link.parent.mkdir(parents=True, exist_ok=True) + if src_link.is_symlink() or src_link.exists(): + src_link.unlink() + src_link.symlink_to(Path("..") / "platform-src" / local_name / "otdfctl") + print(f"Symlinked {src_link} → {otdfctl_dir}") + + return worktree_path diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py index 35e44a1ca..19188b124 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py @@ -3,7 +3,6 @@ from __future__ import annotations import json -import os from typing import Annotated, Any, Optional import typer @@ -113,20 +112,10 @@ def resolve_versions( raise typer.Exit(2) infix = SDK_TAG_INFIXES.get(sdk) - # Set the Go SDK source via OTDFCTL_SOURCE env var - # (standalone otdfctl repo vs platform monorepo). When unset, defaults to standalone. - go_source = os.environ.get("OTDFCTL_SOURCE") if sdk == "go" else None - if go_source and go_source not in ("standalone", "platform"): - typer.echo( - f"Error: unrecognized OTDFCTL_SOURCE={go_source!r}; expected 'platform' or 'standalone'", - err=True, - ) - raise typer.Exit(2) - results: list[ResolveResult] = [] shas: set[str] = set() for version in tags: - v = resolve(sdk, version, infix, go_source=go_source) + v = resolve(sdk, version, infix) if is_resolve_success(v): env = lookup_additional_options(sdk, v["tag"]) if env: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 1046a5ef8..60bc0b730 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -73,8 +73,13 @@ def get_sdk_dirs() -> dict[str, Path]: GO_INSTALL_PREFIX_STANDALONE = "go run github.com/opentdf/otdfctl" GO_INSTALL_PREFIX_PLATFORM = "go run github.com/opentdf/platform/otdfctl" -GO_MODULE_PATH = "github.com/opentdf/otdfctl" -GO_MODULE_PATH_PLATFORM = "github.com/opentdf/platform/otdfctl" +# Module paths keyed by source. "platform" is the canonical, post-migration +# location; "standalone" covers archived pre-v0.31.0 release artifacts that +# remain installable via the Go module proxy. +GO_MODULE_PATHS: dict[str, str] = { + "platform": "github.com/opentdf/platform/otdfctl", + "standalone": "github.com/opentdf/otdfctl", +} LTS_VERSIONS: dict[str, str] = { "go": "0.24.0", @@ -102,9 +107,10 @@ def get_sdk_dirs() -> dict[str, Path]: "0.1.0": "protocol/go/v0.2.3", } -# Bare repo names per SDK (used by checkout) +# Bare repo names per SDK (used by checkout). "go" is intentionally absent — +# otdfctl source builds come from the opentdf/platform monorepo via +# checkout_go_from_platform, not from a standalone clone. SDK_BARE_REPOS: dict[str, str] = { - "go": "otdfctl.git", "java": "java-sdk.git", "js": "web-sdk.git", } @@ -119,42 +125,15 @@ def get_sdk_dirs() -> dict[str, Path]: # (tags are otdfctl/vX.Y.Z in the platform monorepo) SDK_TAG_INFIXES_PLATFORM_GO = "otdfctl" -_VALID_GO_SOURCES = {None, "standalone", "platform"} - - -def _validate_go_source(source: str | None) -> None: - """Raise ValueError if source is not a recognised Go source.""" - if source not in _VALID_GO_SOURCES: - raise ValueError(f"Invalid Go source {source!r}; expected one of {_VALID_GO_SOURCES}") - - -def go_git_url(source: str | None = None) -> str: - """Return the git URL for Go SDK resolution based on source. - - Args: - source: "platform" to use the platform monorepo, None/"standalone" for the - standalone otdfctl repo. - """ - _validate_go_source(source) - if source == "platform": - return SDK_GIT_URLS["platform"] - return SDK_GIT_URLS["go"] - - -def go_tag_infix(source: str | None = None) -> str | None: - """Return the tag infix for Go SDK resolution based on source.""" - _validate_go_source(source) - if source == "platform": - return SDK_TAG_INFIXES_PLATFORM_GO - return None - -def go_module_path(source: str | None = None) -> str: - """Return the Go module path based on source.""" - _validate_go_source(source) - if source == "platform": - return GO_MODULE_PATH_PLATFORM - return GO_MODULE_PATH +def go_module_path(source: str) -> str: + """Return the Go module path for the given source ("platform" or "standalone").""" + try: + return GO_MODULE_PATHS[source] + except KeyError as e: + raise ValueError( + f"Invalid Go source {source!r}; expected one of {sorted(GO_MODULE_PATHS)}" + ) from e ALL_SDKS = ["go", "js", "java"] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 6136d4015..ce0ce2482 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -11,7 +11,7 @@ from pathlib import Path from otdf_sdk_mgr.config import ( - GO_MODULE_PATH_PLATFORM, + GO_MODULE_PATHS, LTS_VERSIONS, get_sdk_dir, get_sdk_dirs, @@ -19,6 +19,7 @@ ) from otdf_sdk_mgr.checkout import checkout_sdk_branch from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions +from otdf_sdk_mgr.resolve import go_source_for from otdf_sdk_mgr.semver import normalize_version @@ -32,13 +33,13 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) The cli.sh and otdfctl.sh wrappers read .version and use `go run @{version}` instead of a local binary. The .version file contains `module-path@version` - (e.g., `github.com/opentdf/otdfctl@v0.24.0`). + (e.g., `github.com/opentdf/platform/otdfctl@v0.31.0`). Args: version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0"). dist_dir: Target distribution directory. - source: "platform" to use the platform monorepo module path, - None or "standalone" for standalone. + source: "platform" or "standalone"; auto-derived from `version` when + omitted (post-v0.31.0 → platform, archived releases → standalone). """ go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) @@ -46,6 +47,8 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) if "/" in version: version = version.rsplit("/", 1)[-1] tag = normalize_version(version) + if source is None: + source = go_source_for(tag) module = go_module_path(source) (dist_dir / ".version").write_text(f"{module}@{tag}\n") shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh") @@ -59,7 +62,7 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) ) if result.returncode != 0: msg = f"go install pre-warm failed: {result.stderr.strip()}" - if module == GO_MODULE_PATH_PLATFORM: + if module == GO_MODULE_PATHS["platform"]: raise InstallError( f"{msg}\nThe platform module path {module}@{tag} may not be published yet." ) @@ -183,8 +186,8 @@ def install_release(sdk: str, version: str, dist_name: str | None = None, **kwar def latest_stable_version(sdk: str) -> tuple[str, str | None] | None: """Find the latest stable version for an SDK that has a CLI available. - Returns (version, source) where source is "platform" for Go versions - from the platform repo, or None otherwise. + Returns (version, source) where source is "platform" or "standalone" for + Go (drives module-path selection in install_go_release), or None otherwise. """ if sdk == "go": versions = list_go_versions() @@ -192,7 +195,7 @@ def latest_stable_version(sdk: str) -> tuple[str, str | None] | None: if not stable: return None entry = stable[-1] - source = "platform" if entry.get("source") == "platform-git-tag" else None + source = "platform" if entry.get("source") == "platform-git-tag" else "standalone" return entry["version"], source elif sdk == "js": versions = list_js_versions() @@ -233,12 +236,21 @@ def cmd_lts(sdks: list[str]) -> None: def cmd_tip(sdks: list[str]) -> None: """Delegate to source checkout + make for head builds.""" + import os + + from otdf_sdk_mgr.checkout import checkout_go_from_platform + sdk_dirs = get_sdk_dirs() for sdk in sdks: print(f"Checking out and building {sdk} from source...") - checkout_sdk_branch(sdk, "main") make_dir = sdk_dirs[sdk] - subprocess.check_call(["make"], cwd=make_dir) + env = os.environ.copy() + if sdk == "go": + platform_dir = checkout_go_from_platform("main") + env["GOWORK"] = str(platform_dir / "go.work") + else: + checkout_sdk_branch(sdk, "main") + subprocess.check_call(["make"], cwd=make_dir, env=env) print(f" {sdk} built from source") diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index 8d27cd68f..9d8445e48 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -70,14 +70,19 @@ def fetch_text(url: str) -> str: def list_go_versions() -> list[dict[str, Any]]: - """List Go SDK versions from git tags in both standalone and platform repos.""" + """List Go SDK versions from git tags in both standalone and platform repos. + + Standalone (opentdf/otdfctl, archived) tags are artifact-install only — + they're flagged ``buildable: False`` and remain available for backward-compat + testing of pre-v0.31.0 releases via the Go module proxy. + """ import git.exc from git import Git repo = Git() seen: dict[str, dict[str, Any]] = {} - # Standalone repo (opentdf/otdfctl): tags like v0.24.0 + # Standalone repo (opentdf/otdfctl): tags like v0.24.0. Archived; artifact-install only. raw = repo.ls_remote(SDK_GIT_URLS["go"], tags=True) for line in raw.strip().split("\n"): if not line: @@ -94,6 +99,7 @@ def list_go_versions() -> list[dict[str, Any]]: "source": "git-tag", "install_method": f"{GO_INSTALL_PREFIX_STANDALONE}@{tag}", "stable": is_stable(tag), + "buildable": False, } # Platform repo (opentdf/platform): tags like otdfctl/v0.X.Y @@ -114,18 +120,14 @@ def list_go_versions() -> list[dict[str, Any]]: continue # Platform entries take precedence (canonical location post-migration); # if the same version exists in both repos, the platform entry - # overwrites the standalone one (a notice is printed below). - if version in seen: - print( - f"Note: version {version} found in both standalone and platform repos; using platform source.", - file=sys.stderr, - ) + # overwrites the standalone one. seen[version] = { "sdk": "go", "version": version, "source": "platform-git-tag", "install_method": f"{GO_INSTALL_PREFIX_PLATFORM}@{version}", "stable": is_stable(version), + "buildable": True, } except git.exc.GitCommandError as e: print( diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index d35609b0d..cf0db9c68 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -13,8 +13,7 @@ LTS_VERSIONS, SDK_GIT_URLS, SDK_NPM_PACKAGES, - go_git_url, - go_tag_infix, + SDK_TAG_INFIXES_PLATFORM_GO, ) @@ -55,6 +54,12 @@ def is_resolve_success(val: ResolveResult) -> TypeGuard[ResolveSuccess]: SHA_REGEX = r"^[a-f0-9]{7,64}$" +# otdfctl moved into the platform monorepo at v0.31.0. Tags below this cannot be +# source-built from the platform checkout (the otdfctl/ subdir doesn't exist at +# those commits) and are resolved against the archived standalone repo for +# artifact install only. +OTDFCTL_PLATFORM_MIN_VERSION = (0, 31, 0) + def _try_resolve_js_npm( sdk: str, @@ -115,44 +120,99 @@ def lookup_additional_options(sdk: str, version: str) -> str | None: return None +def _semver_tuple(version: str) -> tuple[int, int, int] | None: + """Parse 'vX.Y.Z' or 'X.Y.Z' into a tuple; ignore pre-release/build suffixes.""" + m = re.match(r"v?(\d+)\.(\d+)\.(\d+)", version) + if not m: + return None + return (int(m.group(1)), int(m.group(2)), int(m.group(3))) + + +def go_source_for(version: str) -> str: + """Decide whether a go version resolves against platform or standalone. + + Heads, SHAs, branch names, "main", "latest" → platform. Semver tags → + platform for v0.31.0+ (where otdfctl/ lives in the monorepo), else + standalone (artifact-install fallback for archived releases). + "lts" follows LTS_VERSIONS["go"]. + """ + if version in ("main", "latest"): + return "platform" + if re.match(SHA_REGEX, version): + return "platform" + if version.startswith("refs/"): + return "platform" + if version == "lts": + lts_semver = _semver_tuple(LTS_VERSIONS.get("go", "")) + if lts_semver is not None and lts_semver < OTDFCTL_PLATFORM_MIN_VERSION: + return "standalone" + return "platform" + bare = version.removeprefix("otdfctl/") + semver = _semver_tuple(bare) + if semver is not None and semver < OTDFCTL_PLATFORM_MIN_VERSION: + return "standalone" + return "platform" + + +def _looks_like_release_tag(version: str) -> bool: + """Whether `version` could be a release tag (for standalone fallback).""" + bare = version.removeprefix("otdfctl/") + return _semver_tuple(bare) is not None or version == "lts" + + def resolve( sdk: str, version: str, infix: str | None, - go_source: str | None = None, ) -> ResolveResult: """Resolve a version spec to a concrete SHA and tag. + For sdk='go', resolution always targets the platform monorepo's otdfctl/ + subtree (tags `otdfctl/vX.Y.Z`, infix `otdfctl`). Pre-v0.31.0 tags fall + back to the archived standalone opentdf/otdfctl repo and are flagged + artifact-install-only via source="standalone". + Args: sdk: SDK identifier (go, js, java, platform). version: Version spec (main, SHA, tag, latest, lts, etc.). infix: Tag infix for monorepo tag resolution (e.g. "sdk" for JS). - go_source: For sdk=="go", override the git URL and infix. - "platform" resolves against the platform monorepo (otdfctl/ prefix tags). - None or "standalone" uses the standalone otdfctl repo (default). + Ignored for sdk='go' (always overridden to "otdfctl" or None + depending on the resolved source). """ - _go_platform = sdk == "go" and go_source == "platform" - - def _annotate(result: ResolveResult) -> ResolveResult: - """Add source field to successful results when resolving go from platform. - - Error results are returned unchanged. - """ - if _go_platform and is_resolve_success(result): - result["source"] = "platform" + if sdk == "go": + go_source = go_source_for(version) + if go_source == "platform": + sdk_url = SDK_GIT_URLS["platform"] + go_infix: str | None = SDK_TAG_INFIXES_PLATFORM_GO + else: + sdk_url = SDK_GIT_URLS["go"] + go_infix = None + result = _resolve_against(sdk, version, go_infix, sdk_url) + if is_resolve_success(result): + result["source"] = go_source + return result + # Platform miss on a tag-like input: try the archived standalone repo. + if go_source == "platform" and _looks_like_release_tag(version): + fallback = _resolve_against(sdk, version, None, SDK_GIT_URLS["go"]) + if is_resolve_success(fallback): + fallback["source"] = "standalone" + return fallback return result - # Validate config and resolve URLs before the try block so that - # programming errors (bad go_source, unknown SDK) propagate immediately. - if _go_platform: - sdk_url = go_git_url("platform") - infix = go_tag_infix("platform") - else: - try: - sdk_url = SDK_GIT_URLS[sdk] - except KeyError: - return {"sdk": sdk, "alias": version, "err": f"unknown SDK: {sdk}"} + try: + sdk_url = SDK_GIT_URLS[sdk] + except KeyError: + return {"sdk": sdk, "alias": version, "err": f"unknown SDK: {sdk}"} + return _resolve_against(sdk, version, infix, sdk_url) + +def _resolve_against( + sdk: str, + version: str, + infix: str | None, + sdk_url: str, +) -> ResolveResult: + """Run ls-remote-based resolution against a specific git URL.""" try: repo = Git() version = version.removeprefix("refs/heads/") @@ -162,82 +222,70 @@ def _annotate(result: ResolveResult) -> ResolveResult: sha, _ = next(tag for tag in all_heads if "refs/heads/main" in tag) except StopIteration: return {"sdk": sdk, "alias": version, "err": f"main branch not found in {sdk_url}"} - return _annotate( - { - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": "main", - } - ) + return { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": "main", + } if re.match(SHA_REGEX, version): ls_remote = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] matching_tags = [(sha, tag) for (sha, tag) in ls_remote if sha.startswith(version)] if not matching_tags: - return _annotate( - { - "sdk": sdk, - "alias": version[:7], - "sha": version, - "tag": version, - } - ) + return { + "sdk": sdk, + "alias": version[:7], + "sha": version, + "tag": version, + } if len(matching_tags) > 1: for sha, tag in matching_tags: if tag.startswith("refs/pull/"): pr_number = tag.split("/")[2] - return _annotate( - { - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": f"pull-{pr_number}", - } - ) + return { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": f"pull-{pr_number}", + } for sha, tag in matching_tags: mq_match = re.match(MERGE_QUEUE_REGEX, tag) if mq_match: to_branch = mq_match.group("branch") pr_number = mq_match.group("pr_number") if to_branch and pr_number: - return _annotate( - { - "sdk": sdk, - "alias": version, - "head": True, - "pr": pr_number, - "sha": sha, - "tag": f"mq-{to_branch}-{pr_number}", - } - ) - suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] - flattag = "mq--" + suffix.replace("/", "--") - return _annotate( - { + return { "sdk": sdk, "alias": version, "head": True, + "pr": pr_number, "sha": sha, - "tag": flattag, + "tag": f"mq-{to_branch}-{pr_number}", } - ) - head = False - if tag.startswith("refs/heads/"): - head = True - tag = tag.split("refs/heads/")[-1] - flattag = tag.replace("/", "--") - return _annotate( - { + suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] + flattag = "mq--" + suffix.replace("/", "--") + return { "sdk": sdk, "alias": version, - "head": head, + "head": True, "sha": sha, "tag": flattag, } - ) + head = False + if tag.startswith("refs/heads/"): + head = True + tag = tag.split("refs/heads/")[-1] + flattag = tag.replace("/", "--") + return { + "sdk": sdk, + "alias": version, + "head": head, + "sha": sha, + "tag": flattag, + } return { "sdk": sdk, @@ -252,14 +300,12 @@ def _annotate(result: ResolveResult) -> ResolveResult: tag = tag.split("refs/tags/")[-1] if infix: tag = tag.split(f"{infix}/")[-1] - return _annotate( - { - "sdk": sdk, - "alias": version, - "sha": sha, - "tag": tag, - } - ) + return { + "sdk": sdk, + "alias": version, + "sha": sha, + "tag": tag, + } if version.startswith("refs/pull/"): merge_heads = [ @@ -273,16 +319,14 @@ def _annotate(result: ResolveResult) -> ResolveResult: "err": f"pull request {pr_number} not found in {sdk_url}", } sha, _ = merge_heads[0] - return _annotate( - { - "sdk": sdk, - "alias": version, - "head": True, - "pr": pr_number, - "sha": sha, - "tag": f"pull-{pr_number}", - } - ) + return { + "sdk": sdk, + "alias": version, + "head": True, + "pr": pr_number, + "sha": sha, + "tag": f"pull-{pr_number}", + } remote_tags = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] all_listed_tags = [ @@ -297,15 +341,13 @@ def _annotate(result: ResolveResult) -> ResolveResult: if version in all_listed_branches: sha = all_listed_branches[version] - return _annotate( - { - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": version, - } - ) + return { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": version, + } if infix and version.startswith(f"{infix}/"): version = version.split(f"{infix}/")[-1] @@ -349,15 +391,13 @@ def _annotate(result: ResolveResult) -> ResolveResult: if not matching_tags: # No versions with CLI found, fall back to building latest from source sha, tag = stable_tags[-1] - return _annotate( - { - "sdk": sdk, - "alias": alias, - "head": True, # Mark as head to trigger source checkout - "sha": sha, - "tag": tag, - } - ) + return { + "sdk": sdk, + "alias": alias, + "head": True, # Mark as head to trigger source checkout + "sha": sha, + "tag": tag, + } else: matching_tags = stable_tags[-1:] else: @@ -382,15 +422,13 @@ def _annotate(result: ResolveResult) -> ResolveResult: release = tag if infix: release = f"{infix}/{release}" - return _annotate( - { - "sdk": sdk, - "alias": alias, - "release": release, - "sha": sha, - "tag": tag, - } - ) + return { + "sdk": sdk, + "alias": alias, + "release": release, + "sha": sha, + "tag": tag, + } except (git.exc.GitCommandError, ValueError) as e: return { "sdk": sdk, diff --git a/otdf-sdk-mgr/tests/test_resolve.py b/otdf-sdk-mgr/tests/test_resolve.py index 627478c11..a055e78a6 100644 --- a/otdf-sdk-mgr/tests/test_resolve.py +++ b/otdf-sdk-mgr/tests/test_resolve.py @@ -6,6 +6,7 @@ from otdf_sdk_mgr.resolve import ( ResolveResult, _try_resolve_js_npm, + go_source_for, is_resolve_error, is_resolve_success, resolve, @@ -252,15 +253,17 @@ def test_lts_unknown_sdk_raises(self): class TestResolveLatest: def test_non_java_returns_last_stable(self): + # For go, "latest" routes to the platform monorepo (otdfctl/ infix). ls = make_ls_remote( - ("1" * 40, "refs/tags/v0.1.0"), - ("2" * 40, "refs/tags/v0.2.0"), - ("3" * 40, "refs/tags/v0.3.0"), + ("1" * 40, "refs/tags/otdfctl/v0.31.0"), + ("2" * 40, "refs/tags/otdfctl/v0.32.0"), + ("3" * 40, "refs/tags/otdfctl/v0.33.0"), ) with patch_git(ls): result = resolve("go", "latest", None) assert is_resolve_success(result) - assert result["tag"] == "v0.3.0" + assert result["tag"] == "v0.33.0" + assert result.get("source") == "platform" def test_java_with_cli_available(self): ls = make_ls_remote( @@ -302,6 +305,88 @@ def test_java_no_cli_available_falls_back_to_source(self): # --------------------------------------------------------------------------- +class TestGoSourceFor: + """go_source_for() — chooses platform vs standalone for a go version spec.""" + + def test_main_is_platform(self): + assert go_source_for("main") == "platform" + + def test_latest_is_platform(self): + assert go_source_for("latest") == "platform" + + def test_sha_is_platform(self): + assert go_source_for(SHA40) == "platform" + + def test_branch_name_is_platform(self): + assert go_source_for("feature-branch") == "platform" + + def test_post_migration_tag_is_platform(self): + assert go_source_for("v0.31.0") == "platform" + assert go_source_for("v0.32.0") == "platform" + assert go_source_for("otdfctl/v0.32.0") == "platform" + + def test_pre_migration_tag_is_standalone(self): + assert go_source_for("v0.29.0") == "standalone" + assert go_source_for("v0.24.0") == "standalone" + assert go_source_for("0.30.0") == "standalone" + assert go_source_for("otdfctl/v0.29.0") == "standalone" + + def test_lts_follows_lts_versions(self): + # LTS_VERSIONS["go"] is "0.24.0" → standalone. + assert go_source_for("lts") == "standalone" + + +class TestResolveGo: + """resolve() — go-specific routing between platform and standalone.""" + + def test_main_resolves_against_platform_with_source_field(self): + ls = make_ls_remote((SHA40, "refs/heads/main")) + with patch_git(ls): + result = resolve("go", "main", None) + assert is_resolve_success(result) + assert result.get("source") == "platform" + assert result.get("head") is True + + def test_bare_semver_post_migration_routes_to_platform(self): + # Bare "v0.31.0" → looked up as otdfctl/v0.31.0 in platform. + ls = make_ls_remote((SHA40, "refs/tags/otdfctl/v0.31.0")) + with patch_git(ls): + result = resolve("go", "v0.31.0", None) + assert is_resolve_success(result) + assert result["tag"] == "v0.31.0" + assert result["sha"] == SHA40 + assert result.get("source") == "platform" + + def test_prefixed_tag_routes_to_platform(self): + ls = make_ls_remote((SHA40, "refs/tags/otdfctl/v0.32.0")) + with patch_git(ls): + result = resolve("go", "otdfctl/v0.32.0", None) + assert is_resolve_success(result) + assert result["tag"] == "v0.32.0" + assert result.get("source") == "platform" + + def test_pre_migration_tag_routes_to_standalone(self): + # v0.29.0 < 0.31.0 → standalone; mock returns standalone-style tag. + ls = make_ls_remote((SHA40, "refs/tags/v0.29.0")) + with patch_git(ls): + result = resolve("go", "v0.29.0", None) + assert is_resolve_success(result) + assert result["tag"] == "v0.29.0" + assert result.get("source") == "standalone" + + def test_platform_miss_falls_back_to_standalone(self): + # A version that looks post-migration but doesn't exist in platform + # should fall back to standalone if present there. + ls = make_ls_remote((SHA40, "refs/tags/v0.31.99")) + with patch_git(ls): + result = resolve("go", "v0.31.99", None) + assert is_resolve_success(result) + # The fallback resolve runs against the standalone URL with no infix; + # mock returns one tag which matches. + assert result["tag"] == "v0.31.99" + assert result.get("source") == "standalone" + + class TestTryResolveJsNpm: def test_npm_concrete_version(self): tags = [(SHA40, "v1.2.3"), ("0" * 40, "v1.2.2")] diff --git a/xtest/sdk/go/Makefile b/xtest/sdk/go/Makefile index 12b869dc5..24070c4c1 100644 --- a/xtest/sdk/go/Makefile +++ b/xtest/sdk/go/Makefile @@ -20,8 +20,14 @@ build: @echo "Building binaries for versions: $(VERSIONS)" @for version in $(VERSIONS); do \ echo "Building binary for $$version"; \ - cd $(MAKEFILE_DIR)/src/$$version && \ - go build -o $(MAKEFILE_DIR)/binary-$$version . || { \ + src_real="$$(cd $(MAKEFILE_DIR)/src/$$version && pwd -P)"; \ + gowork=""; \ + if [ -z "$$GOWORK" ] && [ -f "$$src_real/../go.work" ]; then \ + gowork="$$(cd $$src_real/.. && pwd -P)/go.work"; \ + echo " Using platform go.work: $$gowork"; \ + fi; \ + (cd "$$src_real" && \ + GOWORK="$${GOWORK:-$$gowork}" go build -o $(MAKEFILE_DIR)/binary-$$version .) || { \ echo "Error: Go build failed for $$version"; \ exit 1; \ }; \ diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 9c446de47..754e14676 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -10,10 +10,11 @@ inputs: required: true platform-otdfctl-dir: description: >- - Absolute path to platform's otdfctl/ directory. When set and sdk is "go", - head versions whose SHA matches platform-otdfctl-sha are symlinked from - here instead of checked out separately. Used in both explicit platform - mode (source: "platform" in resolved version) and auto-detect mode. + Absolute path to platform's otdfctl/ directory. Required when sdk is + "go" and any resolved version has source="platform" (the canonical + path post-otdfctl-migration). When the resolved SHA matches + platform-otdfctl-sha the directory is reused via symlink instead of + a fresh checkout. platform-otdfctl-sha: description: >- SHA of the commit that platform-otdfctl-dir was checked out at. @@ -44,7 +45,10 @@ runs: run: | case "$INPUT_SDK" in "go") - echo "STANDALONE_REPO=opentdf/otdfctl" >> "$GITHUB_ENV" + # otdfctl now lives in the opentdf/platform monorepo (post-v0.31.0); + # the standalone opentdf/otdfctl repo is archived and only used for + # artifact installs of pre-v0.31.0 releases (handled by + # otdf-sdk-mgr install artifact, never a source clone). ;; "java") echo "STANDALONE_REPO=opentdf/java-sdk" >> "$GITHUB_ENV" @@ -157,12 +161,19 @@ runs: is_platform=false use_existing=false - checkout_repo="$STANDALONE_REPO" + # Default checkout target: the SDK's standalone repo. Go has no + # standalone source path (the archived opentdf/otdfctl repo is + # artifact-install-only); the resolver always tags go heads with + # source="platform" so they take the branch below. + checkout_repo="${STANDALONE_REPO:-}" checkout_path="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" if [[ "$source" == "platform" ]]; then - # Explicit platform mode: resolver tagged this version as from - # opentdf/platform. Use per-version SHA to decide checkout strategy. + # otdfctl lives in opentdf/platform/otdfctl; head builds need the + # platform checkout in scope so go.work resolves protocol/go, sdk, + # and lib/* to local sources. Pre-v0.31.0 go releases + # (source="standalone") are installed via Go module proxy and + # never source-checked-out here. is_platform=true if [[ "$needs_source" == "true" && -n "$PLATFORM_OTDFCTL_DIR" \ && -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then @@ -174,13 +185,13 @@ runs: checkout_repo="$PLATFORM_REPO" checkout_path="${INPUT_PATH}/${INPUT_SDK}/platform-src/${tag}" fi - elif [[ "$INPUT_SDK" == "go" && -n "$PLATFORM_OTDFCTL_DIR" && "$needs_source" == "true" ]]; then - # Auto-detect: resolver used standalone repo but detect-otdfctl found - # an embedded otdfctl/ in the platform checkout. PLATFORM_OTDFCTL_DIR - # is only set after confirming the directory exists, so reuse it directly. - echo "::notice::Go version ${tag} resolved from standalone repo; reusing platform-embedded otdfctl at $PLATFORM_OTDFCTL_DIR." - use_existing=true - needs_source=false + elif [[ "$INPUT_SDK" == "go" && "$needs_source" == "true" ]]; then + # Reached when a pre-v0.31.0 (source="standalone") go release's + # artifact install failed; the archived opentdf/otdfctl repo + # offers no source-build fallback. Fail loud rather than letting + # actions/checkout error on an empty repository value. + echo "::error::Go release ${tag} requires source build but pre-v0.31.0 standalone otdfctl source builds are not supported. Check the install-released-versions step log for the underlying go-install failure." + exit 1 fi echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT" From b66994a7b16d2d71b9c46f1872487e03006cc982 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 20 May 2026 16:03:36 -0400 Subject: [PATCH 2/4] fix(otdf-sdk-mgr): handle real dirs and non-branch refs in platform checkout - Remove existing src/ path safely when it's a real directory (not just a symlink), so upgrades from prior layouts don't hard-fail. - Update existing worktrees via fetch + checkout FETCH_HEAD so tags and detached refs work, not just branches. - Accept deprecated otdfctl-source workflow input and emit a warning so callers passing it don't break. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/xtest.yml | 20 ++++++++++++++++++++ otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py | 11 +++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 08dfff506..6b0cddb32 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -28,6 +28,11 @@ on: type: string default: all description: "SDK to focus on (go, js, java, all)" + otdfctl-source: + required: false + type: string + default: "" + description: "DEPRECATED — ignored. otdfctl is always sourced from the opentdf/platform monorepo; pre-v0.31.0 tags fall back to the Go module proxy for artifact install." workflow_call: inputs: platform-ref: @@ -50,6 +55,15 @@ on: required: false type: string default: all + otdfctl-source: + # DEPRECATED — accepted but ignored. otdfctl is always sourced from + # the opentdf/platform monorepo; pre-v0.31.0 tags fall back to the + # Go module proxy for artifact install. Kept declared so existing + # workflow_call sites that pass this input don't fail with + # "Unexpected inputs provided". + required: false + type: string + default: "" schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -89,6 +103,12 @@ jobs: echo "Invalid focus-sdk input: ${FOCUS_SDK_INPUT}. Must be one of: all, go, java, js." >> "$GITHUB_STEP_SUMMARY" exit 1 fi + - name: Warn on deprecated otdfctl-source input + if: ${{ inputs.otdfctl-source != '' }} + env: + OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} + run: |- + echo "::warning::Input 'otdfctl-source=${OTDFCTL_SOURCE_INPUT}' is deprecated and ignored. otdfctl is now always sourced from the opentdf/platform monorepo; pre-v0.31.0 tags fall back to the Go module proxy. Remove this input from your caller." - name: Default Versions depend on context id: default-tags run: |- diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py index b3009ed91..fd9357e77 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/checkout.py @@ -2,6 +2,7 @@ from __future__ import annotations +import shutil import subprocess from pathlib import Path from typing import Any @@ -97,7 +98,8 @@ def checkout_go_from_platform(ref: str) -> Path: if worktree_path.exists(): print(f"Worktree for ref '{ref}' already exists at {worktree_path}. Updating...") - _run(["git", "-C", str(worktree_path), "pull", "origin", ref]) + _run(["git", f"--git-dir={bare_repo_path}", "fetch", "origin", ref, "--tags"]) + _run(["git", "-C", str(worktree_path), "checkout", "--force", "FETCH_HEAD"]) else: print(f"Setting up worktree for ref '{ref}' at {worktree_path}...") _run( @@ -119,8 +121,13 @@ def checkout_go_from_platform(ref: str) -> Path: ) src_link.parent.mkdir(parents=True, exist_ok=True) - if src_link.is_symlink() or src_link.exists(): + if src_link.is_symlink(): src_link.unlink() + elif src_link.exists(): + if src_link.is_dir(): + shutil.rmtree(src_link) + else: + src_link.unlink() src_link.symlink_to(Path("..") / "platform-src" / local_name / "otdfctl") print(f"Symlinked {src_link} → {otdfctl_dir}") From f89b1c86b0596b6165fe4c2b774d40baf6b539f3 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 20 May 2026 21:32:36 -0400 Subject: [PATCH 3/4] refactor(otdf-sdk-mgr): collapse platform/standalone go routing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encode the otdfctl module-path policy in one place (config.go_module_for_tag) and drop the `source` field that was being threaded through resolve → version-info JSON → action.yaml. Pre-v0.31.0 tags use the archived github.com/opentdf/otdfctl module path; everything else uses github.com/opentdf/platform/otdfctl. Resolve always tries the platform monorepo first and falls back once to the archive repo. The CI workflow drops the now-meaningless `otdfctl-source` input and its warning step. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/xtest.yml | 20 ----- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py | 6 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 42 ++++------ otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 87 ++++++-------------- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 72 ++++------------ otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 80 ++---------------- otdf-sdk-mgr/tests/test_resolve.py | 75 ++--------------- xtest/setup-cli-tool/action.yaml | 73 +++++----------- 8 files changed, 92 insertions(+), 363 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 6b0cddb32..08dfff506 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -28,11 +28,6 @@ on: type: string default: all description: "SDK to focus on (go, js, java, all)" - otdfctl-source: - required: false - type: string - default: "" - description: "DEPRECATED — ignored. otdfctl is always sourced from the opentdf/platform monorepo; pre-v0.31.0 tags fall back to the Go module proxy for artifact install." workflow_call: inputs: platform-ref: @@ -55,15 +50,6 @@ on: required: false type: string default: all - otdfctl-source: - # DEPRECATED — accepted but ignored. otdfctl is always sourced from - # the opentdf/platform monorepo; pre-v0.31.0 tags fall back to the - # Go module proxy for artifact install. Kept declared so existing - # workflow_call sites that pass this input don't fail with - # "Unexpected inputs provided". - required: false - type: string - default: "" schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -103,12 +89,6 @@ jobs: echo "Invalid focus-sdk input: ${FOCUS_SDK_INPUT}. Must be one of: all, go, java, js." >> "$GITHUB_STEP_SUMMARY" exit 1 fi - - name: Warn on deprecated otdfctl-source input - if: ${{ inputs.otdfctl-source != '' }} - env: - OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} - run: |- - echo "::warning::Input 'otdfctl-source=${OTDFCTL_SOURCE_INPUT}' is deprecated and ignored. otdfctl is now always sourced from the opentdf/platform monorepo; pre-v0.31.0 tags fall back to the Go module proxy. Remove this input from your caller." - name: Default Versions depend on context id: default-tags run: |- diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py index e62ae2464..e3950d717 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py @@ -74,16 +74,12 @@ def artifact( dist_name: Annotated[ Optional[str], typer.Option("--dist-name", help="Override dist directory name") ] = None, - source: Annotated[ - Optional[str], - typer.Option(help='Source repo for Go CLI (e.g., "platform" for monorepo)'), - ] = None, ) -> None: """Install a single SDK version (used by CI).""" from otdf_sdk_mgr.installers import InstallError, cmd_install try: - cmd_install(sdk, version, dist_name=dist_name, source=source) + cmd_install(sdk, version, dist_name=dist_name) except InstallError as e: typer.echo(f"Error: {e}", err=True) raise typer.Exit(1) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 60bc0b730..e143003e8 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import re from pathlib import Path @@ -70,16 +71,22 @@ def get_sdk_dirs() -> dict[str, Path]: "java": "opentdf/java-sdk", } -GO_INSTALL_PREFIX_STANDALONE = "go run github.com/opentdf/otdfctl" -GO_INSTALL_PREFIX_PLATFORM = "go run github.com/opentdf/platform/otdfctl" +# Canonical post-migration location. +GO_MODULE_PATH_PLATFORM = "github.com/opentdf/platform/otdfctl" +# Archived pre-v0.31.0 releases; still installable via the Go module proxy. +GO_MODULE_PATH_STANDALONE = "github.com/opentdf/otdfctl" -# Module paths keyed by source. "platform" is the canonical, post-migration -# location; "standalone" covers archived pre-v0.31.0 release artifacts that -# remain installable via the Go module proxy. -GO_MODULE_PATHS: dict[str, str] = { - "platform": "github.com/opentdf/platform/otdfctl", - "standalone": "github.com/opentdf/otdfctl", -} +# otdfctl moved into the platform monorepo at v0.31.0; tags below this resolve +# against the archived standalone module path for artifact install only. +OTDFCTL_PLATFORM_MIN_VERSION = (0, 31, 0) + + +def go_module_for_tag(tag: str) -> str: + """Pick the Go module path for an otdfctl release tag.""" + m = re.match(r"v?(\d+)\.(\d+)\.(\d+)", tag) + if m and (int(m.group(1)), int(m.group(2)), int(m.group(3))) < OTDFCTL_PLATFORM_MIN_VERSION: + return GO_MODULE_PATH_STANDALONE + return GO_MODULE_PATH_PLATFORM LTS_VERSIONS: dict[str, str] = { "go": "0.24.0", @@ -108,8 +115,7 @@ def get_sdk_dirs() -> dict[str, Path]: } # Bare repo names per SDK (used by checkout). "go" is intentionally absent — -# otdfctl source builds come from the opentdf/platform monorepo via -# checkout_go_from_platform, not from a standalone clone. +# otdfctl source builds come from opentdf/platform via checkout_go_from_platform. SDK_BARE_REPOS: dict[str, str] = { "java": "java-sdk.git", "js": "web-sdk.git", @@ -121,19 +127,5 @@ def get_sdk_dirs() -> dict[str, Path]: "platform": "service", } -# When resolving go versions from the platform repo, use "otdfctl" infix -# (tags are otdfctl/vX.Y.Z in the platform monorepo) -SDK_TAG_INFIXES_PLATFORM_GO = "otdfctl" - - -def go_module_path(source: str) -> str: - """Return the Go module path for the given source ("platform" or "standalone").""" - try: - return GO_MODULE_PATHS[source] - except KeyError as e: - raise ValueError( - f"Invalid Go source {source!r}; expected one of {sorted(GO_MODULE_PATHS)}" - ) from e - ALL_SDKS = ["go", "js", "java"] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index ce0ce2482..0a2315f2d 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -11,15 +11,14 @@ from pathlib import Path from otdf_sdk_mgr.config import ( - GO_MODULE_PATHS, + GO_MODULE_PATH_PLATFORM, LTS_VERSIONS, get_sdk_dir, get_sdk_dirs, - go_module_path, + go_module_for_tag, ) from otdf_sdk_mgr.checkout import checkout_sdk_branch from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions -from otdf_sdk_mgr.resolve import go_source_for from otdf_sdk_mgr.semver import normalize_version @@ -27,19 +26,13 @@ class InstallError(Exception): """Raised when SDK installation fails.""" -def install_go_release(version: str, dist_dir: Path, source: str | None = None) -> None: +def install_go_release(version: str, dist_dir: Path) -> None: """Install a Go CLI release by writing a .version file. The cli.sh and otdfctl.sh wrappers read .version and use - `go run @{version}` instead of a local binary. - The .version file contains `module-path@version` - (e.g., `github.com/opentdf/platform/otdfctl@v0.31.0`). - - Args: - version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0"). - dist_dir: Target distribution directory. - source: "platform" or "standalone"; auto-derived from `version` when - omitted (post-v0.31.0 → platform, archived releases → standalone). + `go run @{version}` instead of a local binary. The .version file + contains `module-path@version` (e.g., `github.com/opentdf/platform/otdfctl@v0.31.0`). + Pre-v0.31.0 tags use the archived `github.com/opentdf/otdfctl` module path. """ go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) @@ -47,9 +40,7 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) if "/" in version: version = version.rsplit("/", 1)[-1] tag = normalize_version(version) - if source is None: - source = go_source_for(tag) - module = go_module_path(source) + module = go_module_for_tag(tag) (dist_dir / ".version").write_text(f"{module}@{tag}\n") shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh") shutil.copy(go_dir / "otdfctl.sh", dist_dir / "otdfctl.sh") @@ -62,7 +53,7 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) ) if result.returncode != 0: msg = f"go install pre-warm failed: {result.stderr.strip()}" - if module == GO_MODULE_PATHS["platform"]: + if module == GO_MODULE_PATH_PLATFORM: raise InstallError( f"{msg}\nThe platform module path {module}@{tag} may not be published yet." ) @@ -70,7 +61,7 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) print(f" Go release {tag} installed to {dist_dir}") -def install_js_release(version: str, dist_dir: Path, **_kwargs: object) -> None: +def install_js_release(version: str, dist_dir: Path) -> None: """Install a JS CLI release from npm registry.""" js_dir = get_sdk_dir() / "js" dist_dir.mkdir(parents=True, exist_ok=True) @@ -85,7 +76,7 @@ def install_js_release(version: str, dist_dir: Path, **_kwargs: object) -> None: print(f" JS release {v} installed to {dist_dir}") -def install_java_release(version: str, dist_dir: Path, **_kwargs: object) -> None: +def install_java_release(version: str, dist_dir: Path) -> None: """Install a Java CLI release by downloading cmdline.jar from GitHub Releases. Raises InstallError if the artifact is not available or download fails, @@ -153,21 +144,10 @@ def install_java_release(version: str, dist_dir: Path, **_kwargs: object) -> Non } -def install_release(sdk: str, version: str, dist_name: str | None = None, **kwargs: object) -> Path: +def install_release(sdk: str, version: str, dist_name: str | None = None) -> Path: """Install a released version of an SDK CLI. - Args: - sdk: One of "go", "js", "java" - version: Version string (e.g., "v0.24.0" or "0.24.0") - dist_name: Override the dist directory name (defaults to normalized version) - **kwargs: Extra arguments forwarded to the SDK installer - (e.g., source="platform" for Go). - - Returns: - Path to the created dist directory - - Raises: - InstallError: If the SDK is unknown or installation fails. + Raises InstallError if the SDK is unknown or installation fails. """ if sdk not in INSTALLERS: raise InstallError(f"Unknown SDK '{sdk}'. Must be one of: {', '.join(INSTALLERS)}") @@ -179,34 +159,24 @@ def install_release(sdk: str, version: str, dist_name: str | None = None, **kwar print(f" Dist directory already exists: {dist_dir} (skipping)") return dist_dir - INSTALLERS[sdk](version, dist_dir, **kwargs) + INSTALLERS[sdk](version, dist_dir) return dist_dir -def latest_stable_version(sdk: str) -> tuple[str, str | None] | None: - """Find the latest stable version for an SDK that has a CLI available. - - Returns (version, source) where source is "platform" or "standalone" for - Go (drives module-path selection in install_go_release), or None otherwise. - """ +def latest_stable_version(sdk: str) -> str | None: + """Find the latest stable version for an SDK that has a CLI available.""" if sdk == "go": - versions = list_go_versions() - stable = [v for v in versions if v.get("stable", False)] - if not stable: - return None - entry = stable[-1] - source = "platform" if entry.get("source") == "platform-git-tag" else "standalone" - return entry["version"], source + stable = [v for v in list_go_versions() if v.get("stable", False)] + return stable[-1]["version"] if stable else None elif sdk == "js": - versions = list_js_versions() - stable = [v for v in versions if v.get("stable", False)] - return (stable[-1]["version"], None) if stable else None + stable = [v for v in list_js_versions() if v.get("stable", False)] + return stable[-1]["version"] if stable else None elif sdk == "java": - releases = list_java_github_releases() stable_with_cli = [ - v for v in releases if v.get("stable", False) and v.get("has_cli", False) + v for v in list_java_github_releases() + if v.get("stable", False) and v.get("has_cli", False) ] - return (stable_with_cli[-1]["version"], None) if stable_with_cli else None + return stable_with_cli[-1]["version"] if stable_with_cli else None return None @@ -214,13 +184,12 @@ def cmd_stable(sdks: list[str]) -> None: """Install the latest stable release for each SDK.""" for sdk in sdks: print(f"Finding latest stable {sdk} release...") - result = latest_stable_version(sdk) - if result is None: + version = latest_stable_version(sdk) + if version is None: print(f" Warning: No stable version found for {sdk}, skipping") continue - version, source = result print(f" Latest stable {sdk}: {version}") - install_release(sdk, version, source=source) + install_release(sdk, version) def cmd_lts(sdks: list[str]) -> None: @@ -264,9 +233,7 @@ def cmd_release(specs: list[str]) -> None: install_release(sdk, version) -def cmd_install( - sdk: str, version: str, dist_name: str | None = None, source: str | None = None -) -> None: +def cmd_install(sdk: str, version: str, dist_name: str | None = None) -> None: """Install a single SDK version (used by CI action).""" print(f"Installing {sdk} {version}...") - install_release(sdk, version, dist_name=dist_name, source=source) + install_release(sdk, version, dist_name=dist_name) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index 9d8445e48..c4ef9cb49 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -12,13 +12,11 @@ from typing import Any from otdf_sdk_mgr.config import ( - GO_INSTALL_PREFIX_PLATFORM, - GO_INSTALL_PREFIX_STANDALONE, SDK_GITHUB_REPOS, SDK_GIT_URLS, SDK_MAVEN_COORDS, SDK_NPM_PACKAGES, - SDK_TAG_INFIXES_PLATFORM_GO, + go_module_for_tag, ) from otdf_sdk_mgr.semver import is_stable, parse_semver, semver_sort_key @@ -70,20 +68,11 @@ def fetch_text(url: str) -> str: def list_go_versions() -> list[dict[str, Any]]: - """List Go SDK versions from git tags in both standalone and platform repos. - - Standalone (opentdf/otdfctl, archived) tags are artifact-install only — - they're flagged ``buildable: False`` and remain available for backward-compat - testing of pre-v0.31.0 releases via the Go module proxy. - """ - import git.exc + """List Go SDK versions from `otdfctl/vX.Y.Z` tags in opentdf/platform.""" from git import Git - repo = Git() - seen: dict[str, dict[str, Any]] = {} - - # Standalone repo (opentdf/otdfctl): tags like v0.24.0. Archived; artifact-install only. - raw = repo.ls_remote(SDK_GIT_URLS["go"], tags=True) + results: list[dict[str, Any]] = [] + raw = Git().ls_remote(SDK_GIT_URLS["platform"], tags=True) for line in raw.strip().split("\n"): if not line: continue @@ -91,51 +80,18 @@ def list_go_versions() -> list[dict[str, Any]]: if ref.endswith("^{}"): continue tag = ref.removeprefix("refs/tags/") - if not parse_semver(tag): + if not tag.startswith("otdfctl/"): continue - seen[tag] = { + version = tag.removeprefix("otdfctl/") + if not parse_semver(version): + continue + results.append({ "sdk": "go", - "version": tag, - "source": "git-tag", - "install_method": f"{GO_INSTALL_PREFIX_STANDALONE}@{tag}", - "stable": is_stable(tag), - "buildable": False, - } - - # Platform repo (opentdf/platform): tags like otdfctl/v0.X.Y - infix = SDK_TAG_INFIXES_PLATFORM_GO - try: - raw = repo.ls_remote(SDK_GIT_URLS["platform"], tags=True) - for line in raw.strip().split("\n"): - if not line: - continue - _, ref = line.split("\t", 1) - if ref.endswith("^{}"): - continue - tag = ref.removeprefix("refs/tags/") - if not tag.startswith(f"{infix}/"): - continue - version = tag.removeprefix(f"{infix}/") - if not parse_semver(version): - continue - # Platform entries take precedence (canonical location post-migration); - # if the same version exists in both repos, the platform entry - # overwrites the standalone one. - seen[version] = { - "sdk": "go", - "version": version, - "source": "platform-git-tag", - "install_method": f"{GO_INSTALL_PREFIX_PLATFORM}@{version}", - "stable": is_stable(version), - "buildable": True, - } - except git.exc.GitCommandError as e: - print( - f"::warning::Failed to query platform repo for go tags: {e}", - file=sys.stderr, - ) - - results = list(seen.values()) + "version": version, + "source": "platform-git-tag", + "install_method": f"go run {go_module_for_tag(version)}@{version}", + "stable": is_stable(version), + }) results.sort(key=lambda r: semver_sort_key(r["version"])) return results diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index cf0db9c68..ca026344c 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -13,7 +13,6 @@ LTS_VERSIONS, SDK_GIT_URLS, SDK_NPM_PACKAGES, - SDK_TAG_INFIXES_PLATFORM_GO, ) @@ -25,7 +24,6 @@ class ResolveSuccess(TypedDict): pr: NotRequired[str] release: NotRequired[str] sha: str - source: NotRequired[str] tag: str @@ -54,12 +52,6 @@ def is_resolve_success(val: ResolveResult) -> TypeGuard[ResolveSuccess]: SHA_REGEX = r"^[a-f0-9]{7,64}$" -# otdfctl moved into the platform monorepo at v0.31.0. Tags below this cannot be -# source-built from the platform checkout (the otdfctl/ subdir doesn't exist at -# those commits) and are resolved against the archived standalone repo for -# artifact install only. -OTDFCTL_PLATFORM_MIN_VERSION = (0, 31, 0) - def _try_resolve_js_npm( sdk: str, @@ -120,46 +112,6 @@ def lookup_additional_options(sdk: str, version: str) -> str | None: return None -def _semver_tuple(version: str) -> tuple[int, int, int] | None: - """Parse 'vX.Y.Z' or 'X.Y.Z' into a tuple; ignore pre-release/build suffixes.""" - m = re.match(r"v?(\d+)\.(\d+)\.(\d+)", version) - if not m: - return None - return (int(m.group(1)), int(m.group(2)), int(m.group(3))) - - -def go_source_for(version: str) -> str: - """Decide whether a go version resolves against platform or standalone. - - Heads, SHAs, branch names, "main", "latest" → platform. Semver tags → - platform for v0.31.0+ (where otdfctl/ lives in the monorepo), else - standalone (artifact-install fallback for archived releases). - "lts" follows LTS_VERSIONS["go"]. - """ - if version in ("main", "latest"): - return "platform" - if re.match(SHA_REGEX, version): - return "platform" - if version.startswith("refs/"): - return "platform" - if version == "lts": - lts_semver = _semver_tuple(LTS_VERSIONS.get("go", "")) - if lts_semver is not None and lts_semver < OTDFCTL_PLATFORM_MIN_VERSION: - return "standalone" - return "platform" - bare = version.removeprefix("otdfctl/") - semver = _semver_tuple(bare) - if semver is not None and semver < OTDFCTL_PLATFORM_MIN_VERSION: - return "standalone" - return "platform" - - -def _looks_like_release_tag(version: str) -> bool: - """Whether `version` could be a release tag (for standalone fallback).""" - bare = version.removeprefix("otdfctl/") - return _semver_tuple(bare) is not None or version == "lts" - - def resolve( sdk: str, version: str, @@ -167,37 +119,15 @@ def resolve( ) -> ResolveResult: """Resolve a version spec to a concrete SHA and tag. - For sdk='go', resolution always targets the platform monorepo's otdfctl/ - subtree (tags `otdfctl/vX.Y.Z`, infix `otdfctl`). Pre-v0.31.0 tags fall - back to the archived standalone opentdf/otdfctl repo and are flagged - artifact-install-only via source="standalone". - - Args: - sdk: SDK identifier (go, js, java, platform). - version: Version spec (main, SHA, tag, latest, lts, etc.). - infix: Tag infix for monorepo tag resolution (e.g. "sdk" for JS). - Ignored for sdk='go' (always overridden to "otdfctl" or None - depending on the resolved source). + For sdk='go', resolution targets the platform monorepo's otdfctl/ subtree + (tags `otdfctl/vX.Y.Z`). Tags not found there fall back to the archived + opentdf/otdfctl repo (pre-v0.31.0). The `infix` argument is ignored for go. """ if sdk == "go": - go_source = go_source_for(version) - if go_source == "platform": - sdk_url = SDK_GIT_URLS["platform"] - go_infix: str | None = SDK_TAG_INFIXES_PLATFORM_GO - else: - sdk_url = SDK_GIT_URLS["go"] - go_infix = None - result = _resolve_against(sdk, version, go_infix, sdk_url) + result = _resolve_against("go", version, "otdfctl", SDK_GIT_URLS["platform"]) if is_resolve_success(result): - result["source"] = go_source return result - # Platform miss on a tag-like input: try the archived standalone repo. - if go_source == "platform" and _looks_like_release_tag(version): - fallback = _resolve_against(sdk, version, None, SDK_GIT_URLS["go"]) - if is_resolve_success(fallback): - fallback["source"] = "standalone" - return fallback - return result + return _resolve_against("go", version, None, SDK_GIT_URLS["go"]) try: sdk_url = SDK_GIT_URLS[sdk] diff --git a/otdf-sdk-mgr/tests/test_resolve.py b/otdf-sdk-mgr/tests/test_resolve.py index a055e78a6..ddf16e082 100644 --- a/otdf-sdk-mgr/tests/test_resolve.py +++ b/otdf-sdk-mgr/tests/test_resolve.py @@ -6,7 +6,6 @@ from otdf_sdk_mgr.resolve import ( ResolveResult, _try_resolve_js_npm, - go_source_for, is_resolve_error, is_resolve_success, resolve, @@ -263,7 +262,6 @@ def test_non_java_returns_last_stable(self): result = resolve("go", "latest", None) assert is_resolve_success(result) assert result["tag"] == "v0.33.0" - assert result.get("source") == "platform" def test_java_with_cli_available(self): ls = make_ls_remote( @@ -305,86 +303,31 @@ def test_java_no_cli_available_falls_back_to_source(self): # --------------------------------------------------------------------------- -class TestGoSourceFor: - """go_source_for() — chooses platform vs standalone for a go version spec.""" - - def test_main_is_platform(self): - assert go_source_for("main") == "platform" - - def test_latest_is_platform(self): - assert go_source_for("latest") == "platform" - - def test_sha_is_platform(self): - assert go_source_for(SHA40) == "platform" - - def test_branch_name_is_platform(self): - assert go_source_for("feature-branch") == "platform" - - def test_post_migration_tag_is_platform(self): - assert go_source_for("v0.31.0") == "platform" - assert go_source_for("v0.32.0") == "platform" - assert go_source_for("otdfctl/v0.32.0") == "platform" - - def test_pre_migration_tag_is_standalone(self): - assert go_source_for("v0.29.0") == "standalone" - assert go_source_for("v0.24.0") == "standalone" - assert go_source_for("0.30.0") == "standalone" - assert go_source_for("otdfctl/v0.29.0") == "standalone" - - def test_lts_follows_lts_versions(self): - # LTS_VERSIONS["go"] is "0.24.0" → standalone. - assert go_source_for("lts") == "standalone" - - class TestResolveGo: - """resolve() — go-specific routing between platform and standalone.""" + """resolve() — go always tries the platform monorepo first.""" - def test_main_resolves_against_platform_with_source_field(self): + def test_main_resolves_against_platform(self): ls = make_ls_remote((SHA40, "refs/heads/main")) with patch_git(ls): result = resolve("go", "main", None) assert is_resolve_success(result) - assert result.get("source") == "platform" assert result.get("head") is True - def test_bare_semver_post_migration_routes_to_platform(self): - # Bare "v0.31.0" → looked up as otdfctl/v0.31.0 in platform. - ls = make_ls_remote((SHA40, "refs/tags/otdfctl/v0.31.0")) - with patch_git(ls): - result = resolve("go", "v0.31.0", None) - assert is_resolve_success(result) - assert result["tag"] == "v0.31.0" - assert result["sha"] == SHA40 - assert result.get("source") == "platform" - - def test_prefixed_tag_routes_to_platform(self): + def test_prefixed_tag_resolves_in_platform(self): ls = make_ls_remote((SHA40, "refs/tags/otdfctl/v0.32.0")) with patch_git(ls): result = resolve("go", "otdfctl/v0.32.0", None) assert is_resolve_success(result) assert result["tag"] == "v0.32.0" - assert result.get("source") == "platform" - - def test_pre_migration_tag_routes_to_standalone(self): - # v0.29.0 < 0.31.0 → standalone; mock returns standalone-style tag. - ls = make_ls_remote((SHA40, "refs/tags/v0.29.0")) - with patch_git(ls): - result = resolve("go", "v0.29.0", None) - assert is_resolve_success(result) - assert result["tag"] == "v0.29.0" - assert result.get("source") == "standalone" - def test_platform_miss_falls_back_to_standalone(self): - # A version that looks post-migration but doesn't exist in platform - # should fall back to standalone if present there. - ls = make_ls_remote((SHA40, "refs/tags/v0.31.99")) + def test_platform_miss_falls_back_to_archive(self): + # Tags not present in opentdf/platform (e.g. pre-v0.31.0 releases) + # fall back to the archived opentdf/otdfctl repo. + ls = make_ls_remote((SHA40, "refs/tags/v0.24.0")) with patch_git(ls): - result = resolve("go", "v0.31.99", None) + result = resolve("go", "v0.24.0", None) assert is_resolve_success(result) - # The fallback resolve runs against the standalone URL with no infix; - # mock returns one tag which matches. - assert result["tag"] == "v0.31.99" - assert result.get("source") == "standalone" + assert result["tag"] == "v0.24.0" class TestTryResolveJsNpm: diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 754e14676..c03295d04 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -10,16 +10,12 @@ inputs: required: true platform-otdfctl-dir: description: >- - Absolute path to platform's otdfctl/ directory. Required when sdk is - "go" and any resolved version has source="platform" (the canonical - path post-otdfctl-migration). When the resolved SHA matches - platform-otdfctl-sha the directory is reused via symlink instead of - a fresh checkout. + Absolute path to platform's otdfctl/ directory. When set and a resolved + go version's SHA matches platform-otdfctl-sha the directory is reused + via symlink instead of a fresh checkout. platform-otdfctl-sha: description: >- SHA of the commit that platform-otdfctl-dir was checked out at. - Used to decide which Go head version (if any) can reuse the existing - platform checkout vs needing a fresh one. outputs: version-a: description: "Object containing tag, sha, and name of a version checked out" @@ -43,19 +39,13 @@ runs: - name: identify repo urls shell: bash run: | + # Go source builds always come from opentdf/platform (otdfctl moved + # there at v0.31.0); pre-v0.31.0 tags are artifact-install-only via + # the Go module proxy and never reach the source-checkout path here. case "$INPUT_SDK" in - "go") - # otdfctl now lives in the opentdf/platform monorepo (post-v0.31.0); - # the standalone opentdf/otdfctl repo is archived and only used for - # artifact installs of pre-v0.31.0 releases (handled by - # otdf-sdk-mgr install artifact, never a source clone). - ;; - "java") - echo "STANDALONE_REPO=opentdf/java-sdk" >> "$GITHUB_ENV" - ;; - "js") - echo "STANDALONE_REPO=opentdf/web-sdk" >> "$GITHUB_ENV" - ;; + "go") ;; + "java") echo "STANDALONE_REPO=opentdf/java-sdk" >> "$GITHUB_ENV" ;; + "js") echo "STANDALONE_REPO=opentdf/web-sdk" >> "$GITHUB_ENV" ;; *) echo "Invalid SDK specified: $INPUT_SDK" >> "$GITHUB_STEP_SUMMARY" exit 1 @@ -105,16 +95,10 @@ runs: release=$(echo "$row" | jq -r '.release // empty') if [[ "$head" != "true" && -n "$release" ]]; then echo "Installing $INPUT_SDK $tag from registry (release: $release)" - # Sanitize tag for use as an env var name (replace non-alphanumeric/underscore with _) tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" - source=$(echo "$row" | jq -r '.source // empty') - source_args=() - [[ -n "$source" ]] && source_args=(--source "$source") if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \ - --sdk "$INPUT_SDK" --version "$release" \ - --dist-name "$tag" "${source_args[@]}"; then - echo " Warning: Artifact installation failed for $INPUT_SDK $tag" - echo " Will fall back to building from source" + --sdk "$INPUT_SDK" --version "$release" --dist-name "$tag"; then + echo " Warning: Artifact installation failed for $INPUT_SDK $tag; falling back to source" echo "BUILD_FROM_SOURCE_${tag_sanitized}=true" >> "$GITHUB_ENV" fi fi @@ -128,10 +112,10 @@ runs: id: check-source shell: bash run: | - # Determine which version slots need source checkout and from which repo. - # A slot needs checkout if it is a head version OR if artifact install failed. - # Platform-source versions may reuse the existing platform-otdfctl-dir - # (when their SHA matches) or need a fresh opentdf/platform checkout. + # A slot needs checkout if it is a head version OR if artifact install + # failed. For go, the only source path is opentdf/platform's otdfctl/ + # subtree — reuse the existing platform checkout when the SHAs match, + # otherwise check out a fresh one. for slot in a b c d; do case "$slot" in a) row=$(echo "${version_info}" | jq -rc '.[0] // empty') ;; @@ -151,7 +135,6 @@ runs: tag=$(echo "$row" | jq -r '.tag') head=$(echo "$row" | jq -r '.head // false') sha=$(echo "$row" | jq -r '.sha') - source=$(echo "$row" | jq -r '.source // empty') tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" build_from_source_var="BUILD_FROM_SOURCE_${tag_sanitized}" needs_source=false @@ -161,37 +144,19 @@ runs: is_platform=false use_existing=false - # Default checkout target: the SDK's standalone repo. Go has no - # standalone source path (the archived opentdf/otdfctl repo is - # artifact-install-only); the resolver always tags go heads with - # source="platform" so they take the branch below. checkout_repo="${STANDALONE_REPO:-}" checkout_path="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" - if [[ "$source" == "platform" ]]; then - # otdfctl lives in opentdf/platform/otdfctl; head builds need the - # platform checkout in scope so go.work resolves protocol/go, sdk, - # and lib/* to local sources. Pre-v0.31.0 go releases - # (source="standalone") are installed via Go module proxy and - # never source-checked-out here. + if [[ "$INPUT_SDK" == "go" && "$needs_source" == "true" ]]; then is_platform=true - if [[ "$needs_source" == "true" && -n "$PLATFORM_OTDFCTL_DIR" \ - && -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then - # SHA matches existing platform checkout — reuse via symlink + if [[ -n "$PLATFORM_OTDFCTL_DIR" && -n "$PLATFORM_OTDFCTL_SHA" \ + && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then use_existing=true needs_source=false - elif [[ "$needs_source" == "true" ]]; then - # Different SHA — need a fresh platform checkout + else checkout_repo="$PLATFORM_REPO" checkout_path="${INPUT_PATH}/${INPUT_SDK}/platform-src/${tag}" fi - elif [[ "$INPUT_SDK" == "go" && "$needs_source" == "true" ]]; then - # Reached when a pre-v0.31.0 (source="standalone") go release's - # artifact install failed; the archived opentdf/otdfctl repo - # offers no source-build fallback. Fail loud rather than letting - # actions/checkout error on an empty repository value. - echo "::error::Go release ${tag} requires source build but pre-v0.31.0 standalone otdfctl source builds are not supported. Check the install-released-versions step log for the underlying go-install failure." - exit 1 fi echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT" From e6ac04cc06108974a7a36fd489e425f99b506316 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 20 May 2026 21:58:56 -0400 Subject: [PATCH 4/4] fixup ruff format --- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 1 + otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 3 ++- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 16 +++++++++------- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index e143003e8..d24063e8f 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -88,6 +88,7 @@ def go_module_for_tag(tag: str) -> str: return GO_MODULE_PATH_STANDALONE return GO_MODULE_PATH_PLATFORM + LTS_VERSIONS: dict[str, str] = { "go": "0.24.0", "java": "0.9.0", diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 0a2315f2d..f3c11b7da 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -173,7 +173,8 @@ def latest_stable_version(sdk: str) -> str | None: return stable[-1]["version"] if stable else None elif sdk == "java": stable_with_cli = [ - v for v in list_java_github_releases() + v + for v in list_java_github_releases() if v.get("stable", False) and v.get("has_cli", False) ] return stable_with_cli[-1]["version"] if stable_with_cli else None diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index c4ef9cb49..187e08fe9 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -85,13 +85,15 @@ def list_go_versions() -> list[dict[str, Any]]: version = tag.removeprefix("otdfctl/") if not parse_semver(version): continue - results.append({ - "sdk": "go", - "version": version, - "source": "platform-git-tag", - "install_method": f"go run {go_module_for_tag(version)}@{version}", - "stable": is_stable(version), - }) + results.append( + { + "sdk": "go", + "version": version, + "source": "platform-git-tag", + "install_method": f"go run {go_module_for_tag(version)}@{version}", + "stable": is_stable(version), + } + ) results.sort(key=lambda r: semver_sort_key(r["version"])) return results