From 3567fd36b7f34c36a6032af835304aaf9abc9f06 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 15 Apr 2026 16:29:32 -0400 Subject: [PATCH 01/17] feat(xtest): support platform-embedded otdfctl for migration to monorepo otdfctl is moving from opentdf/otdfctl into opentdf/platform. This updates the test infrastructure to auto-detect when the platform checkout contains otdfctl/ and build from there instead of the standalone repo. Key changes: - xtest.yml: new otdfctl-source input (auto/standalone/platform) and detection step that checks for otdfctl/go.mod in the platform dir - setup-cli-tool: new platform-otdfctl-dir input; symlinks platform source into sdk/go/src/ for head builds instead of separate checkout - otdf-sdk-mgr: resolve.py supports go_source="platform" to resolve against the platform repo with otdfctl/ tag infix; installers write .module-path file for the new Go module path - cli.sh/otdfctl.sh: read .module-path to use the correct module path (github.com/opentdf/platform/otdfctl) for go run fallback Backward compatible: old releases still resolve from the standalone repo; .module-path absence defaults to the original module path. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 46 ++++++++++- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py | 6 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 41 ++++++++++ otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 20 +++-- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 79 +++++++++++++------ xtest/sdk/go/cli.sh | 8 +- xtest/sdk/go/otdfctl.sh | 8 +- xtest/setup-cli-tool/action.yaml | 55 +++++++++++-- 8 files changed, 219 insertions(+), 44 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 773448892..e8dc257b0 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -28,6 +28,11 @@ on: type: string default: all description: "SDK to focus on (go, js, java, all)" + otdfctl-source: + required: false + type: string + default: auto + description: "otdfctl source: 'auto' (detect from platform checkout), 'standalone', or 'platform'" workflow_call: inputs: platform-ref: @@ -50,6 +55,10 @@ on: required: false type: string default: all + otdfctl-source: + required: false + type: string + default: auto schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -198,8 +207,9 @@ jobs: const tagToSha = {}; const headTags = []; - for (const { tag, head, sha, alias, err, release } of refInfo) { - const sdkRepoUrl = `https://github.com/opentdf/${encodeURIComponent(sdkType == 'js' ? 'web-sdk' : sdkType == 'go' ? 'otdfctl' : sdkType == 'java' ? 'java-sdk' : sdkType)}`; + for (const { tag, head, sha, alias, err, release, source } of refInfo) { + const goRepoName = source === 'platform' ? 'platform' : 'otdfctl'; + const sdkRepoUrl = `https://github.com/opentdf/${encodeURIComponent(sdkType == 'js' ? 'web-sdk' : sdkType == 'go' ? goRepoName : sdkType == 'java' ? 'java-sdk' : sdkType)}`; const sdkLink = `${htmlEscape(sdkType)}`; const commitLink = sha ? `${htmlEscape(sha.substring(0, 7))}` : ' . '; const tagLink = (release && tag) @@ -290,6 +300,29 @@ jobs: with: node-version: "22.x" + ######## DETECT PLATFORM-EMBEDDED OTDFCTL ############# + - name: Detect platform-embedded otdfctl + id: detect-otdfctl + run: | + PLATFORM_DIR="${{ steps.run-platform.outputs.platform-working-dir }}" + if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then + if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" + echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" + echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + else + echo "otdfctl not found in platform checkout; using standalone repo" + echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + fi + elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then + echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" + echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + else + echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + fi + env: + OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} + ######### CHECKOUT JS CLI ############# - name: Configure js-sdk id: configure-js @@ -324,6 +357,7 @@ jobs: path: otdftests/xtest/sdk sdk: go version-info: "${{ needs.resolve-versions.outputs.go }}" + platform-otdfctl-dir: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} - name: Cache Go modules if: fromJson(steps.configure-go.outputs.heads)[0] != null @@ -345,11 +379,15 @@ jobs: OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }} - name: Replace otdfctl go.mod packages, but only at head version of platform - if: fromJson(steps.configure-go.outputs.heads)[0] != null && env.FOCUS_SDK == 'go' && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) + if: >- + steps.detect-otdfctl.outputs.otdfctl-source != 'platform' + && fromJson(steps.configure-go.outputs.heads)[0] != null + && env.FOCUS_SDK == 'go' + && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) env: PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} run: |- - echo "Replacing go.mod packages..." + echo "Replacing go.mod packages (standalone otdfctl)..." PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}" OTDFCTL_DIR_ABS="$(pwd)/otdftests/xtest/sdk/go/src/" echo "PLATFORM_DIR_ABS: $PLATFORM_DIR_ABS" diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py index 19188b124..1d4d6823a 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os from typing import Annotated, Any, Optional import typer @@ -112,10 +113,13 @@ def resolve_versions( raise typer.Exit(2) infix = SDK_TAG_INFIXES.get(sdk) + # Allow overriding the Go SDK source (standalone otdfctl repo vs platform monorepo) + go_source = os.environ.get("OTDFCTL_SOURCE") if sdk == "go" else None + results: list[ResolveResult] = [] shas: set[str] = set() for version in tags: - v = resolve(sdk, version, infix) + v = resolve(sdk, version, infix, go_source=go_source) if is_resolve_success(v): env = lookup_additional_options(sdk, v["tag"]) if env: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index adf6c8b1f..25b1b1195 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -71,6 +71,10 @@ def get_sdk_dirs() -> dict[str, Path]: } GO_INSTALL_PREFIX = "go run github.com/opentdf/otdfctl" +GO_INSTALL_PREFIX_PLATFORM = "go run github.com/opentdf/platform/otdfctl" + +GO_MODULE_PATH = "github.com/opentdf/otdfctl" +GO_MODULE_PATH_PLATFORM = "github.com/opentdf/platform/otdfctl" LTS_VERSIONS: dict[str, str] = { "go": "0.24.0", @@ -111,4 +115,41 @@ def get_sdk_dirs() -> dict[str, Path]: "platform": "service", } +# When resolving go versions from the platform repo, use "otdfctl" infix +# (tags are otdfctl/v0.X.Y in the platform monorepo) +SDK_TAG_INFIXES_PLATFORM_GO = "otdfctl" + + +def go_git_url(source: str | None = None) -> str: + """Return the git URL for Go SDK resolution based on source. + + Args: + source: "platform" to use the platform monorepo, None/"standalone" for the + standalone otdfctl repo. + """ + if source == "platform": + return SDK_GIT_URLS["platform"] + return SDK_GIT_URLS["go"] + + +def go_tag_infix(source: str | None = None) -> str | None: + """Return the tag infix for Go SDK resolution based on source.""" + if source == "platform": + return SDK_TAG_INFIXES_PLATFORM_GO + return None + + +def go_install_prefix(source: str | None = None) -> str: + """Return the go install/run prefix based on source.""" + if source == "platform": + return GO_INSTALL_PREFIX_PLATFORM + return GO_INSTALL_PREFIX + + +def go_module_path(source: str | None = None) -> str: + """Return the Go module path based on source.""" + if source == "platform": + return GO_MODULE_PATH_PLATFORM + return GO_MODULE_PATH + ALL_SDKS = ["go", "js", "java"] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index e7c22ae09..395b771fa 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -11,9 +11,11 @@ from pathlib import Path from otdf_sdk_mgr.config import ( + GO_MODULE_PATH_PLATFORM, LTS_VERSIONS, get_sdk_dir, get_sdk_dirs, + go_install_prefix, ) from otdf_sdk_mgr.checkout import checkout_sdk_branch from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions @@ -24,22 +26,30 @@ class InstallError(Exception): """Raised when SDK installation fails.""" -def install_go_release(version: str, dist_dir: Path) -> None: +def install_go_release(version: str, dist_dir: Path, source: str | None = None) -> None: """Install a Go CLI release by writing a .version file. - The cli.sh and otdfctl.sh wrappers read .version and use - `go run github.com/opentdf/otdfctl@{version}` instead of a local binary. + The cli.sh and otdfctl.sh wrappers read .version (and optionally .module-path) + and use `go run @{version}` instead of a local binary. + + Args: + version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0"). + dist_dir: Target distribution directory. + source: "platform" to use the platform monorepo module path, None for standalone. """ go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) tag = normalize_version(version) (dist_dir / ".version").write_text(f"{tag}\n") + if source == "platform": + (dist_dir / ".module-path").write_text(f"{GO_MODULE_PATH_PLATFORM}\n") shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh") shutil.copy(go_dir / "otdfctl.sh", dist_dir / "otdfctl.sh") shutil.copy(go_dir / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml") - print(f" Pre-warming Go cache for otdfctl@{tag}...") + install_module = go_install_prefix(source).removeprefix("go run ") + print(f" Pre-warming Go cache for {install_module}@{tag}...") result = subprocess.run( - ["go", "install", f"github.com/opentdf/otdfctl@{tag}"], + ["go", "install", f"{install_module}@{tag}"], capture_output=True, text=True, ) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 6e4cd7ca1..0c8ad5e9e 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -12,6 +12,8 @@ LTS_VERSIONS, SDK_GIT_URLS, SDK_NPM_PACKAGES, + go_git_url, + go_tag_infix, ) @@ -23,6 +25,7 @@ class ResolveSuccess(TypedDict): pr: NotRequired[str] release: NotRequired[str] sha: str + source: NotRequired[str] tag: str @@ -111,78 +114,104 @@ def lookup_additional_options(sdk: str, version: str) -> str | None: return None -def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: - """Resolve a version spec to a concrete SHA and tag.""" +def resolve( + sdk: str, + version: str, + infix: str | None, + go_source: str | None = None, +) -> ResolveResult: + """Resolve a version spec to a concrete SHA and tag. + + Args: + sdk: SDK identifier (go, js, java, platform). + version: Version spec (main, SHA, tag, latest, lts, etc.). + infix: Tag infix for monorepo tag resolution (e.g. "sdk" for JS). + go_source: For sdk=="go", override the git URL and infix. + "platform" resolves against the platform monorepo (otdfctl/ prefix tags). + None or "standalone" uses the standalone otdfctl repo (default). + """ + _go_platform = sdk == "go" and go_source == "platform" + + def _annotate(result: ResolveResult) -> ResolveResult: + """Add source field to successful results when resolving go from platform.""" + if _go_platform and is_resolve_success(result): + result["source"] = "platform" + return result + try: - sdk_url = SDK_GIT_URLS[sdk] + if _go_platform: + sdk_url = go_git_url("platform") + infix = go_tag_infix("platform") + else: + sdk_url = SDK_GIT_URLS[sdk] repo = Git() if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0] - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "sha": sha, "tag": "main", - } + }) if re.match(SHA_REGEX, version): ls_remote = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] matching_tags = [(sha, tag) for (sha, tag) in ls_remote if sha.startswith(version)] if not matching_tags: - return { + return _annotate({ "sdk": sdk, "alias": version[:7], "sha": version, "tag": version, - } + }) if len(matching_tags) > 1: for sha, tag in matching_tags: if tag.startswith("refs/pull/"): pr_number = tag.split("/")[2] - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "sha": sha, "tag": f"pull-{pr_number}", - } + }) for sha, tag in matching_tags: mq_match = re.match(MERGE_QUEUE_REGEX, tag) if mq_match: to_branch = mq_match.group("branch") pr_number = mq_match.group("pr_number") if to_branch and pr_number: - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "pr": pr_number, "sha": sha, "tag": f"mq-{to_branch}-{pr_number}", - } + }) suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] flattag = "mq--" + suffix.replace("/", "--") - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "sha": sha, "tag": flattag, - } + }) head = False if tag.startswith("refs/heads/"): head = True tag = tag.split("refs/heads/")[-1] flattag = tag.replace("/", "--") - return { + return _annotate({ "sdk": sdk, "alias": version, "head": head, "sha": sha, "tag": flattag, - } + }) return { "sdk": sdk, @@ -197,12 +226,12 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: tag = tag.split("refs/tags/")[-1] if infix: tag = tag.split(f"{infix}/")[-1] - return { + return _annotate({ "sdk": sdk, "alias": version, "sha": sha, "tag": tag, - } + }) if version.startswith("refs/pull/"): merge_heads = [ @@ -216,14 +245,14 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: "err": f"pull request {pr_number} not found in {sdk_url}", } sha, _ = merge_heads[0] - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "pr": pr_number, "sha": sha, "tag": f"pull-{pr_number}", - } + }) remote_tags = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] all_listed_tags = [ @@ -238,13 +267,13 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: if version in all_listed_branches: sha = all_listed_branches[version] - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "sha": sha, "tag": version, - } + }) if infix and version.startswith(f"{infix}/"): version = version.split(f"{infix}/")[-1] @@ -288,13 +317,13 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: if not matching_tags: # No versions with CLI found, fall back to building latest from source sha, tag = stable_tags[-1] - return { + return _annotate({ "sdk": sdk, "alias": alias, "head": True, # Mark as head to trigger source checkout "sha": sha, "tag": tag, - } + }) else: matching_tags = stable_tags[-1:] else: @@ -319,13 +348,13 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: release = tag if infix: release = f"{infix}/{release}" - return { + return _annotate({ "sdk": sdk, "alias": alias, "release": release, "sha": sha, "tag": tag, - } + }) except Exception as e: return { "sdk": sdk, diff --git a/xtest/sdk/go/cli.sh b/xtest/sdk/go/cli.sh index 172aa5b50..18c2ee735 100755 --- a/xtest/sdk/go/cli.sh +++ b/xtest/sdk/go/cli.sh @@ -22,11 +22,15 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then + MODULE_PATH="github.com/opentdf/otdfctl" + if [ -f "$SCRIPT_DIR/.module-path" ]; then + MODULE_PATH=$(tr -d '[:space:]' <"$SCRIPT_DIR/.module-path") + fi if [ -f "$SCRIPT_DIR/.version" ]; then OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") - cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}") + cmd=(go run "${MODULE_PATH}@${OTDFCTL_VERSION}") else - cmd=(go run "github.com/opentdf/otdfctl@latest") + cmd=(go run "${MODULE_PATH}@latest") fi fi diff --git a/xtest/sdk/go/otdfctl.sh b/xtest/sdk/go/otdfctl.sh index 17fbb0c84..0b3a404d0 100755 --- a/xtest/sdk/go/otdfctl.sh +++ b/xtest/sdk/go/otdfctl.sh @@ -17,11 +17,15 @@ source "$XTEST_DIR/test.env" cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then + MODULE_PATH="github.com/opentdf/otdfctl" + if [ -f "$SCRIPT_DIR/.module-path" ]; then + MODULE_PATH=$(tr -d '[:space:]' <"$SCRIPT_DIR/.module-path") + fi if [ -f "$SCRIPT_DIR/.version" ]; then OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") - cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}") + cmd=(go run "${MODULE_PATH}@${OTDFCTL_VERSION}") else - cmd=(go run "github.com/opentdf/otdfctl@latest") + cmd=(go run "${MODULE_PATH}@latest") fi fi diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 9e110ef4f..bee697707 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -8,6 +8,11 @@ inputs: version-info: description: JSON-encoded output of otdf-sdk-mgr versions resolve required: true + platform-otdfctl-dir: + description: >- + Absolute path to platform's otdfctl/ directory. When set and sdk is "go", + head versions are symlinked from here instead of checked out from the + standalone opentdf/otdfctl repo. outputs: version-a: description: "Object containing tag, sha, and name of a version checked out" @@ -33,19 +38,30 @@ runs: run: | case "${{ inputs.sdk }}" in "go") - echo "sdk_repo=opentdf/otdfctl" >> $GITHUB_ENV + if [[ -n "$PLATFORM_OTDFCTL_DIR" && -f "$PLATFORM_OTDFCTL_DIR/go.mod" ]]; then + echo "sdk_repo=opentdf/platform" >> $GITHUB_ENV + echo "sdk_source=platform" >> $GITHUB_ENV + echo "Using platform-embedded otdfctl from $PLATFORM_OTDFCTL_DIR" + else + echo "sdk_repo=opentdf/otdfctl" >> $GITHUB_ENV + echo "sdk_source=standalone" >> $GITHUB_ENV + fi ;; "java") echo "sdk_repo=opentdf/java-sdk" >> $GITHUB_ENV + echo "sdk_source=standalone" >> $GITHUB_ENV ;; "js") echo "sdk_repo=opentdf/web-sdk" >> $GITHUB_ENV + echo "sdk_source=standalone" >> $GITHUB_ENV ;; *) echo "Invalid SDK specified: ${{ inputs.sdk }}" >> $GITHUB_STEP_SUMMARY exit 1 ;; esac + env: + PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} - name: resolve versions id: resolve @@ -131,10 +147,36 @@ runs: env: version_info: ${{ inputs.version-info }} + - name: symlink platform-embedded otdfctl for head versions + if: env.sdk_source == 'platform' + shell: bash + run: | + # When otdfctl lives inside the platform repo, symlink it into + # sdk/go/src/{tag} so the Makefile builds it the same way. + for slot in a b c d; do + case "$slot" in + a) version_json='${{ steps.resolve.outputs.version-a }}' ; needs_source='${{ steps.check-source.outputs.needs-source-a }}' ;; + b) version_json='${{ steps.resolve.outputs.version-b }}' ; needs_source='${{ steps.check-source.outputs.needs-source-b }}' ;; + c) version_json='${{ steps.resolve.outputs.version-c }}' ; needs_source='${{ steps.check-source.outputs.needs-source-c }}' ;; + d) version_json='${{ steps.resolve.outputs.version-d }}' ; needs_source='${{ steps.check-source.outputs.needs-source-d }}' ;; + esac + if [[ -z "$version_json" || "$needs_source" != "true" ]]; then + continue + fi + tag=$(echo "$version_json" | jq -r '.tag') + src_dir="${{ inputs.path }}/${{ inputs.sdk }}/src/${tag}" + echo "Symlinking platform otdfctl to ${src_dir}" + mkdir -p "$(dirname "$src_dir")" + ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir" + done + env: + PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} + - name: checkout version a uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - steps.resolve.outputs.version-a != '' + env.sdk_source != 'platform' + && steps.resolve.outputs.version-a != '' && steps.check-source.outputs.needs-source-a == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-a).tag }} @@ -145,7 +187,8 @@ runs: - name: checkout version b uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - steps.resolve.outputs.version-b != '' + env.sdk_source != 'platform' + && steps.resolve.outputs.version-b != '' && steps.check-source.outputs.needs-source-b == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-b).tag }} @@ -156,7 +199,8 @@ runs: - name: checkout version c uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - steps.resolve.outputs.version-c != '' + env.sdk_source != 'platform' + && steps.resolve.outputs.version-c != '' && steps.check-source.outputs.needs-source-c == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-c).tag }} @@ -167,7 +211,8 @@ runs: - name: checkout version d uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - steps.resolve.outputs.version-d != '' + env.sdk_source != 'platform' + && steps.resolve.outputs.version-d != '' && steps.check-source.outputs.needs-source-d == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-d).tag }} From 0ea2e5d0290d87bb4b28cf356aa1a744b2041a8b Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 08:27:13 -0400 Subject: [PATCH 02/17] fixup ruff format --- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 1 + otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 172 +++++++++++++---------- 2 files changed, 98 insertions(+), 75 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 25b1b1195..a7416f618 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -152,4 +152,5 @@ def go_module_path(source: str | None = None) -> str: return GO_MODULE_PATH_PLATFORM return GO_MODULE_PATH + ALL_SDKS = ["go", "js", "java"] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 0c8ad5e9e..a180c60c2 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -148,70 +148,82 @@ def _annotate(result: ResolveResult) -> ResolveResult: if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0] - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": "main", - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": "main", + } + ) if re.match(SHA_REGEX, version): ls_remote = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] matching_tags = [(sha, tag) for (sha, tag) in ls_remote if sha.startswith(version)] if not matching_tags: - return _annotate({ - "sdk": sdk, - "alias": version[:7], - "sha": version, - "tag": version, - }) + return _annotate( + { + "sdk": sdk, + "alias": version[:7], + "sha": version, + "tag": version, + } + ) if len(matching_tags) > 1: for sha, tag in matching_tags: if tag.startswith("refs/pull/"): pr_number = tag.split("/")[2] - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": f"pull-{pr_number}", - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": f"pull-{pr_number}", + } + ) for sha, tag in matching_tags: mq_match = re.match(MERGE_QUEUE_REGEX, tag) if mq_match: to_branch = mq_match.group("branch") pr_number = mq_match.group("pr_number") if to_branch and pr_number: - return _annotate({ + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "pr": pr_number, + "sha": sha, + "tag": f"mq-{to_branch}-{pr_number}", + } + ) + suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] + flattag = "mq--" + suffix.replace("/", "--") + return _annotate( + { "sdk": sdk, "alias": version, "head": True, - "pr": pr_number, "sha": sha, - "tag": f"mq-{to_branch}-{pr_number}", - }) - suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] - flattag = "mq--" + suffix.replace("/", "--") - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": flattag, - }) + "tag": flattag, + } + ) head = False if tag.startswith("refs/heads/"): head = True tag = tag.split("refs/heads/")[-1] flattag = tag.replace("/", "--") - return _annotate({ - "sdk": sdk, - "alias": version, - "head": head, - "sha": sha, - "tag": flattag, - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": head, + "sha": sha, + "tag": flattag, + } + ) return { "sdk": sdk, @@ -226,12 +238,14 @@ def _annotate(result: ResolveResult) -> ResolveResult: tag = tag.split("refs/tags/")[-1] if infix: tag = tag.split(f"{infix}/")[-1] - return _annotate({ - "sdk": sdk, - "alias": version, - "sha": sha, - "tag": tag, - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "sha": sha, + "tag": tag, + } + ) if version.startswith("refs/pull/"): merge_heads = [ @@ -245,14 +259,16 @@ def _annotate(result: ResolveResult) -> ResolveResult: "err": f"pull request {pr_number} not found in {sdk_url}", } sha, _ = merge_heads[0] - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "pr": pr_number, - "sha": sha, - "tag": f"pull-{pr_number}", - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "pr": pr_number, + "sha": sha, + "tag": f"pull-{pr_number}", + } + ) remote_tags = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] all_listed_tags = [ @@ -267,13 +283,15 @@ def _annotate(result: ResolveResult) -> ResolveResult: if version in all_listed_branches: sha = all_listed_branches[version] - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": version, - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": version, + } + ) if infix and version.startswith(f"{infix}/"): version = version.split(f"{infix}/")[-1] @@ -317,13 +335,15 @@ def _annotate(result: ResolveResult) -> ResolveResult: if not matching_tags: # No versions with CLI found, fall back to building latest from source sha, tag = stable_tags[-1] - return _annotate({ - "sdk": sdk, - "alias": alias, - "head": True, # Mark as head to trigger source checkout - "sha": sha, - "tag": tag, - }) + return _annotate( + { + "sdk": sdk, + "alias": alias, + "head": True, # Mark as head to trigger source checkout + "sha": sha, + "tag": tag, + } + ) else: matching_tags = stable_tags[-1:] else: @@ -348,13 +368,15 @@ def _annotate(result: ResolveResult) -> ResolveResult: release = tag if infix: release = f"{infix}/{release}" - return _annotate({ - "sdk": sdk, - "alias": alias, - "release": release, - "sha": sha, - "tag": tag, - }) + return _annotate( + { + "sdk": sdk, + "alias": alias, + "release": release, + "sha": sha, + "tag": tag, + } + ) except Exception as e: return { "sdk": sdk, From 7c20d73a1961c6dced8e2e9ea71bb39d036ffcc4 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 08:45:09 -0400 Subject: [PATCH 03/17] fix(xtest): update remaining otdfctl references for platform monorepo migration - artifactLink in xtest.yml now uses the correct pkg.go.dev module path based on the source field (platform vs standalone) - registry.py list_go_versions() queries both the standalone otdfctl repo and the platform repo (otdfctl/ prefixed tags), deduplicating with platform entries taking precedence - README.md documents both module paths for Go release installs Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 10 +++-- otdf-sdk-mgr/README.md | 2 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 4 +- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 48 ++++++++++++++++++----- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index e8dc257b0..f1b4d1eff 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -179,7 +179,7 @@ jobs: core.summary.addHeading('Versions under Test', 3); - function artifactLink(sdkType, tag, release, head) { + function artifactLink(sdkType, tag, release, head, source) { if (head || !release) return ''; const v = tag.replace(/^v/, ''); if (sdkType === 'js') { @@ -191,7 +191,11 @@ jobs: return `Maven Central`; } if (sdkType === 'go') { - const url = `https://pkg.go.dev/github.com/opentdf/otdfctl@${encodeURIComponent(tag)}`; + const modulePath = source === 'platform' + ? `github.com/opentdf/platform/otdfctl` + : `github.com/opentdf/otdfctl`; + const moduleTag = source === 'platform' ? `otdfctl/${tag}` : tag; + const url = `https://pkg.go.dev/${modulePath}@${encodeURIComponent(moduleTag)}`; return `pkg.go.dev`; } return ''; @@ -215,7 +219,7 @@ jobs: const tagLink = (release && tag) ? `${htmlEscape(tag)}` : tag ? htmlEscape(tag) : 'N/A'; - const artifactCell = artifactLink(sdkType, tag, release, head); + const artifactCell = artifactLink(sdkType, tag, release, head, source); table.push([sdkLink, tagLink, commitLink, alias || 'N/A', artifactCell || 'N/A', err || 'N/A']); if (err) { errorCount += 1; diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index ee2a2a8b7..2d765241d 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -56,7 +56,7 @@ otdf-sdk-mgr java-fixup ## How Release Installs Work -- **Go**: Writes a `.version` file; `cli.sh`/`otdfctl.sh` use `go run github.com/opentdf/otdfctl@{version}` (no local compilation needed, Go caches the binary) +- **Go**: Writes a `.version` file (and optionally `.module-path`); `cli.sh`/`otdfctl.sh` use `go run @{version}` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. - **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/` - **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar` diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index a7416f618..1ef2979d2 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -70,7 +70,7 @@ def get_sdk_dirs() -> dict[str, Path]: "java": "opentdf/java-sdk", } -GO_INSTALL_PREFIX = "go run github.com/opentdf/otdfctl" +GO_INSTALL_PREFIX_STANDALONE = "go run github.com/opentdf/otdfctl" GO_INSTALL_PREFIX_PLATFORM = "go run github.com/opentdf/platform/otdfctl" GO_MODULE_PATH = "github.com/opentdf/otdfctl" @@ -143,7 +143,7 @@ def go_install_prefix(source: str | None = None) -> str: """Return the go install/run prefix based on source.""" if source == "platform": return GO_INSTALL_PREFIX_PLATFORM - return GO_INSTALL_PREFIX + return GO_INSTALL_PREFIX_STANDALONE def go_module_path(source: str | None = None) -> str: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index 8f8dd34e5..aae9a4eeb 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -12,11 +12,13 @@ from typing import Any from otdf_sdk_mgr.config import ( - GO_INSTALL_PREFIX, + GO_INSTALL_PREFIX_PLATFORM, + GO_INSTALL_PREFIX_STANDALONE, SDK_GITHUB_REPOS, SDK_GIT_URLS, SDK_MAVEN_COORDS, SDK_NPM_PACKAGES, + SDK_TAG_INFIXES_PLATFORM_GO, ) from otdf_sdk_mgr.semver import is_stable, parse_semver, semver_sort_key @@ -68,12 +70,14 @@ def fetch_text(url: str) -> str: def list_go_versions() -> list[dict[str, Any]]: - """List Go SDK versions from git tags.""" + """List Go SDK versions from git tags in both standalone and platform repos.""" from git import Git repo = Git() + seen: dict[str, dict[str, Any]] = {} + + # Standalone repo (opentdf/otdfctl): tags like v0.24.0 raw = repo.ls_remote(SDK_GIT_URLS["go"], tags=True) - results = [] for line in raw.strip().split("\n"): if not line: continue @@ -83,16 +87,42 @@ def list_go_versions() -> list[dict[str, Any]]: tag = ref.removeprefix("refs/tags/") if not parse_semver(tag): continue - version = tag - results.append( - { + seen[tag] = { + "sdk": "go", + "version": tag, + "source": "git-tag", + "install_method": f"{GO_INSTALL_PREFIX_STANDALONE}@{tag}", + "stable": is_stable(tag), + } + + # Platform repo (opentdf/platform): tags like otdfctl/v0.X.Y + infix = SDK_TAG_INFIXES_PLATFORM_GO + try: + raw = repo.ls_remote(SDK_GIT_URLS["platform"], tags=True) + for line in raw.strip().split("\n"): + if not line: + continue + _, ref = line.split("\t", 1) + if ref.endswith("^{}"): + continue + tag = ref.removeprefix("refs/tags/") + if not tag.startswith(f"{infix}/"): + continue + version = tag.removeprefix(f"{infix}/") + if not parse_semver(version): + continue + # Platform entries take precedence (canonical location post-migration) + seen[version] = { "sdk": "go", "version": version, - "source": "git-tag", - "install_method": f"{GO_INSTALL_PREFIX}@{version}", + "source": "platform-git-tag", + "install_method": f"{GO_INSTALL_PREFIX_PLATFORM}@{tag}", "stable": is_stable(version), } - ) + except Exception as e: + print(f"Warning: failed to query platform repo for go tags: {e}", file=sys.stderr) + + results = list(seen.values()) results.sort(key=lambda r: semver_sort_key(r["version"])) return results From cd0aec513965bae93d1b42871ece729b3189ac1e Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 08:46:09 -0400 Subject: [PATCH 04/17] fixup pkg.go removes tag prefixes IIRC --- .github/workflows/xtest.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index f1b4d1eff..c1b9b6746 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -194,8 +194,7 @@ jobs: const modulePath = source === 'platform' ? `github.com/opentdf/platform/otdfctl` : `github.com/opentdf/otdfctl`; - const moduleTag = source === 'platform' ? `otdfctl/${tag}` : tag; - const url = `https://pkg.go.dev/${modulePath}@${encodeURIComponent(moduleTag)}`; + const url = `https://pkg.go.dev/${modulePath}@${encodeURIComponent(tag)}`; return `pkg.go.dev`; } return ''; From 1f027001ebe8660b701ef966596841edff80d8d5 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 09:00:41 -0400 Subject: [PATCH 05/17] refactor(xtest): consolidate .version and .module-path into single .version file The Go SDK version config now uses a single .version file with format `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`) instead of separate .version and .module-path files. Shell wrappers fall back to the standalone module path for legacy bare-version files. Co-Authored-By: Claude Opus 4.6 (1M context) --- otdf-sdk-mgr/README.md | 2 +- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 19 +++++++++---------- xtest/sdk/go/cli.sh | 16 +++++++++------- xtest/sdk/go/otdfctl.sh | 16 +++++++++------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index 2d765241d..818ee4226 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -56,7 +56,7 @@ otdf-sdk-mgr java-fixup ## How Release Installs Work -- **Go**: Writes a `.version` file (and optionally `.module-path`); `cli.sh`/`otdfctl.sh` use `go run @{version}` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. +- **Go**: Writes a `.version` file containing `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`); `cli.sh`/`otdfctl.sh` use `go run @{version}` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. - **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/` - **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar` diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 395b771fa..cb2173800 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -11,11 +11,10 @@ from pathlib import Path from otdf_sdk_mgr.config import ( - GO_MODULE_PATH_PLATFORM, LTS_VERSIONS, get_sdk_dir, get_sdk_dirs, - go_install_prefix, + go_module_path, ) from otdf_sdk_mgr.checkout import checkout_sdk_branch from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions @@ -29,8 +28,10 @@ class InstallError(Exception): def install_go_release(version: str, dist_dir: Path, source: str | None = None) -> None: """Install a Go CLI release by writing a .version file. - The cli.sh and otdfctl.sh wrappers read .version (and optionally .module-path) - and use `go run @{version}` instead of a local binary. + The cli.sh and otdfctl.sh wrappers read .version and use + `go run @{version}` instead of a local binary. + The .version file contains `module-path@version` + (e.g., `github.com/opentdf/otdfctl@v0.24.0`). Args: version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0"). @@ -40,16 +41,14 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) tag = normalize_version(version) - (dist_dir / ".version").write_text(f"{tag}\n") - if source == "platform": - (dist_dir / ".module-path").write_text(f"{GO_MODULE_PATH_PLATFORM}\n") + module = go_module_path(source) + (dist_dir / ".version").write_text(f"{module}@{tag}\n") shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh") shutil.copy(go_dir / "otdfctl.sh", dist_dir / "otdfctl.sh") shutil.copy(go_dir / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml") - install_module = go_install_prefix(source).removeprefix("go run ") - print(f" Pre-warming Go cache for {install_module}@{tag}...") + print(f" Pre-warming Go cache for {module}@{tag}...") result = subprocess.run( - ["go", "install", f"{install_module}@{tag}"], + ["go", "install", f"{module}@{tag}"], capture_output=True, text=True, ) diff --git a/xtest/sdk/go/cli.sh b/xtest/sdk/go/cli.sh index 18c2ee735..f97b20c1a 100755 --- a/xtest/sdk/go/cli.sh +++ b/xtest/sdk/go/cli.sh @@ -22,15 +22,17 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then - MODULE_PATH="github.com/opentdf/otdfctl" - if [ -f "$SCRIPT_DIR/.module-path" ]; then - MODULE_PATH=$(tr -d '[:space:]' <"$SCRIPT_DIR/.module-path") - fi if [ -f "$SCRIPT_DIR/.version" ]; then - OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") - cmd=(go run "${MODULE_PATH}@${OTDFCTL_VERSION}") + VERSION_SPEC=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") + if [[ "$VERSION_SPEC" == *@* ]]; then + # New format: module-path@version + cmd=(go run "$VERSION_SPEC") + else + # Legacy format: bare version tag, default to standalone module + cmd=(go run "github.com/opentdf/otdfctl@${VERSION_SPEC}") + fi else - cmd=(go run "${MODULE_PATH}@latest") + cmd=(go run "github.com/opentdf/otdfctl@latest") fi fi diff --git a/xtest/sdk/go/otdfctl.sh b/xtest/sdk/go/otdfctl.sh index 0b3a404d0..9ba55f054 100755 --- a/xtest/sdk/go/otdfctl.sh +++ b/xtest/sdk/go/otdfctl.sh @@ -17,15 +17,17 @@ source "$XTEST_DIR/test.env" cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then - MODULE_PATH="github.com/opentdf/otdfctl" - if [ -f "$SCRIPT_DIR/.module-path" ]; then - MODULE_PATH=$(tr -d '[:space:]' <"$SCRIPT_DIR/.module-path") - fi if [ -f "$SCRIPT_DIR/.version" ]; then - OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") - cmd=(go run "${MODULE_PATH}@${OTDFCTL_VERSION}") + VERSION_SPEC=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") + if [[ "$VERSION_SPEC" == *@* ]]; then + # New format: module-path@version + cmd=(go run "$VERSION_SPEC") + else + # Legacy format: bare version tag, default to standalone module + cmd=(go run "github.com/opentdf/otdfctl@${VERSION_SPEC}") + fi else - cmd=(go run "${MODULE_PATH}@latest") + cmd=(go run "github.com/opentdf/otdfctl@latest") fi fi From 9b9c9d78ae3328f5a9b64d1070cf560c24540f4d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 09:13:27 -0400 Subject: [PATCH 06/17] feat(sdk-mgr): wire --source option through install artifact command Allows specifying the source repo for Go CLI installs (e.g., --source platform) to support the otdfctl migration from standalone repo to the platform monorepo. Co-Authored-By: Claude Opus 4.6 (1M context) --- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py | 6 +++++- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py index e3950d717..e62ae2464 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py @@ -74,12 +74,16 @@ def artifact( dist_name: Annotated[ Optional[str], typer.Option("--dist-name", help="Override dist directory name") ] = None, + source: Annotated[ + Optional[str], + typer.Option(help='Source repo for Go CLI (e.g., "platform" for monorepo)'), + ] = None, ) -> None: """Install a single SDK version (used by CI).""" from otdf_sdk_mgr.installers import InstallError, cmd_install try: - cmd_install(sdk, version, dist_name=dist_name) + cmd_install(sdk, version, dist_name=dist_name, source=source) except InstallError as e: typer.echo(f"Error: {e}", err=True) raise typer.Exit(1) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index cb2173800..dc36ad11a 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -59,7 +59,7 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) print(f" Go release {tag} installed to {dist_dir}") -def install_js_release(version: str, dist_dir: Path) -> None: +def install_js_release(version: str, dist_dir: Path, **_kwargs: object) -> None: """Install a JS CLI release from npm registry.""" js_dir = get_sdk_dir() / "js" dist_dir.mkdir(parents=True, exist_ok=True) @@ -74,7 +74,7 @@ def install_js_release(version: str, dist_dir: Path) -> None: print(f" JS release {v} installed to {dist_dir}") -def install_java_release(version: str, dist_dir: Path) -> None: +def install_java_release(version: str, dist_dir: Path, **_kwargs: object) -> None: """Install a Java CLI release by downloading cmdline.jar from GitHub Releases. Raises InstallError if the artifact is not available or download fails, @@ -142,13 +142,17 @@ def install_java_release(version: str, dist_dir: Path) -> None: } -def install_release(sdk: str, version: str, dist_name: str | None = None) -> Path: +def install_release( + sdk: str, version: str, dist_name: str | None = None, **kwargs: object +) -> Path: """Install a released version of an SDK CLI. Args: sdk: One of "go", "js", "java" version: Version string (e.g., "v0.24.0" or "0.24.0") dist_name: Override the dist directory name (defaults to normalized version) + **kwargs: Extra arguments forwarded to the SDK installer + (e.g., source="platform" for Go). Returns: Path to the created dist directory @@ -166,7 +170,7 @@ def install_release(sdk: str, version: str, dist_name: str | None = None) -> Pat print(f" Dist directory already exists: {dist_dir} (skipping)") return dist_dir - INSTALLERS[sdk](version, dist_dir) + INSTALLERS[sdk](version, dist_dir, **kwargs) return dist_dir @@ -233,7 +237,9 @@ def cmd_release(specs: list[str]) -> None: install_release(sdk, version) -def cmd_install(sdk: str, version: str, dist_name: str | None = None) -> None: +def cmd_install( + sdk: str, version: str, dist_name: str | None = None, source: str | None = None +) -> None: """Install a single SDK version (used by CI action).""" print(f"Installing {sdk} {version}...") - install_release(sdk, version, dist_name=dist_name) + install_release(sdk, version, dist_name=dist_name, source=source) From 0c1b428c19ff253aead560d593d869d257c6be82 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 09:18:40 -0400 Subject: [PATCH 07/17] fix(setup-cli-tool): avoid script injection by using env vars for inputs and step outputs Co-Authored-By: Claude Sonnet 4.6 --- xtest/setup-cli-tool/action.yaml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index bee697707..eb992db2e 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -155,22 +155,32 @@ runs: # sdk/go/src/{tag} so the Makefile builds it the same way. for slot in a b c d; do case "$slot" in - a) version_json='${{ steps.resolve.outputs.version-a }}' ; needs_source='${{ steps.check-source.outputs.needs-source-a }}' ;; - b) version_json='${{ steps.resolve.outputs.version-b }}' ; needs_source='${{ steps.check-source.outputs.needs-source-b }}' ;; - c) version_json='${{ steps.resolve.outputs.version-c }}' ; needs_source='${{ steps.check-source.outputs.needs-source-c }}' ;; - d) version_json='${{ steps.resolve.outputs.version-d }}' ; needs_source='${{ steps.check-source.outputs.needs-source-d }}' ;; + a) version_json="$VERSION_A" ; needs_source="$NEEDS_SOURCE_A" ;; + b) version_json="$VERSION_B" ; needs_source="$NEEDS_SOURCE_B" ;; + c) version_json="$VERSION_C" ; needs_source="$NEEDS_SOURCE_C" ;; + d) version_json="$VERSION_D" ; needs_source="$NEEDS_SOURCE_D" ;; esac if [[ -z "$version_json" || "$needs_source" != "true" ]]; then continue fi tag=$(echo "$version_json" | jq -r '.tag') - src_dir="${{ inputs.path }}/${{ inputs.sdk }}/src/${tag}" + src_dir="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" echo "Symlinking platform otdfctl to ${src_dir}" mkdir -p "$(dirname "$src_dir")" ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir" done env: PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} + INPUT_PATH: ${{ inputs.path }} + INPUT_SDK: ${{ inputs.sdk }} + VERSION_A: ${{ steps.resolve.outputs.version-a }} + VERSION_B: ${{ steps.resolve.outputs.version-b }} + VERSION_C: ${{ steps.resolve.outputs.version-c }} + VERSION_D: ${{ steps.resolve.outputs.version-d }} + NEEDS_SOURCE_A: ${{ steps.check-source.outputs.needs-source-a }} + NEEDS_SOURCE_B: ${{ steps.check-source.outputs.needs-source-b }} + NEEDS_SOURCE_C: ${{ steps.check-source.outputs.needs-source-c }} + NEEDS_SOURCE_D: ${{ steps.check-source.outputs.needs-source-d }} - name: checkout version a uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 From bf739dc58bd22ee5ab4db27f53f0f9f35e4fa751 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 09:07:39 -0400 Subject: [PATCH 08/17] Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index a180c60c2..8403759eb 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -147,7 +147,7 @@ def _annotate(result: ResolveResult) -> ResolveResult: repo = Git() if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] - sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0] + sha, _ = next(tag for tag in all_heads if "refs/heads/main" in tag) return _annotate( { "sdk": sdk, From edb5e3a37d48fc3ea129fb1d3ef2007cd743f541 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 10:59:36 -0400 Subject: [PATCH 09/17] feat(setup-cli-tool): support multiple platform-source Go versions Replace global sdk_source/sdk_repo with per-version checkout decisions so that multiple Go head versions from opentdf/platform at different SHAs can be tested against each other (e.g. a platform PR vs main). - Add platform-otdfctl-sha input for SHA-based matching of the existing platform checkout; versions with a different SHA get a fresh checkout of opentdf/platform into platform-src/{tag}/ with a symlink from otdfctl/ into src/{tag}/ - Wire OTDFCTL_SOURCE to the resolve-versions job when otdfctl-source is explicitly 'platform', so the resolver returns platform SHAs - Preserve backward-compatible auto-detect behavior: when platform- otdfctl-dir is set but version-info lacks source:"platform", all head Go versions still symlink to the existing dir Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 6 + xtest/setup-cli-tool/action.yaml | 202 ++++++++++++++++++++++--------- 2 files changed, 149 insertions(+), 59 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index c1b9b6746..58eec1342 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -87,6 +87,9 @@ jobs: JS_REF: "${{ inputs.js-ref }}" OTDFCTL_REF: "${{ inputs.otdfctl-ref }}" JAVA_REF: "${{ inputs.java-ref }}" + # When explicitly set to 'platform', tells the Go resolver to resolve + # against opentdf/platform tags instead of the standalone otdfctl repo. + OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input if: ${{ inputs.focus-sdk != '' }} @@ -313,6 +316,7 @@ jobs: echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" else echo "otdfctl not found in platform checkout; using standalone repo" echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" @@ -320,6 +324,7 @@ jobs: elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" else echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" fi @@ -361,6 +366,7 @@ jobs: sdk: go version-info: "${{ needs.resolve-versions.outputs.go }}" platform-otdfctl-dir: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} + platform-otdfctl-sha: ${{ steps.detect-otdfctl.outputs.otdfctl-sha }} - name: Cache Go modules if: fromJson(steps.configure-go.outputs.heads)[0] != null diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index eb992db2e..1aa56d986 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -11,8 +11,13 @@ inputs: platform-otdfctl-dir: description: >- Absolute path to platform's otdfctl/ directory. When set and sdk is "go", - head versions are symlinked from here instead of checked out from the - standalone opentdf/otdfctl repo. + head versions whose SHA matches platform-otdfctl-sha are symlinked from + here instead of checked out separately. + platform-otdfctl-sha: + description: >- + SHA of the commit that platform-otdfctl-dir was checked out at. + Used to decide which Go head version (if any) can reuse the existing + platform checkout vs needing a fresh one. outputs: version-a: description: "Object containing tag, sha, and name of a version checked out" @@ -33,35 +38,27 @@ outputs: runs: using: composite steps: - - name: identify repo url + - name: identify repo urls shell: bash run: | - case "${{ inputs.sdk }}" in + case "$INPUT_SDK" in "go") - if [[ -n "$PLATFORM_OTDFCTL_DIR" && -f "$PLATFORM_OTDFCTL_DIR/go.mod" ]]; then - echo "sdk_repo=opentdf/platform" >> $GITHUB_ENV - echo "sdk_source=platform" >> $GITHUB_ENV - echo "Using platform-embedded otdfctl from $PLATFORM_OTDFCTL_DIR" - else - echo "sdk_repo=opentdf/otdfctl" >> $GITHUB_ENV - echo "sdk_source=standalone" >> $GITHUB_ENV - fi + echo "STANDALONE_REPO=opentdf/otdfctl" >> "$GITHUB_ENV" ;; "java") - echo "sdk_repo=opentdf/java-sdk" >> $GITHUB_ENV - echo "sdk_source=standalone" >> $GITHUB_ENV + echo "STANDALONE_REPO=opentdf/java-sdk" >> "$GITHUB_ENV" ;; "js") - echo "sdk_repo=opentdf/web-sdk" >> $GITHUB_ENV - echo "sdk_source=standalone" >> $GITHUB_ENV + echo "STANDALONE_REPO=opentdf/web-sdk" >> "$GITHUB_ENV" ;; *) - echo "Invalid SDK specified: ${{ inputs.sdk }}" >> $GITHUB_STEP_SUMMARY + echo "Invalid SDK specified: $INPUT_SDK" >> "$GITHUB_STEP_SUMMARY" exit 1 ;; esac + echo "PLATFORM_REPO=opentdf/platform" >> "$GITHUB_ENV" env: - PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} + INPUT_SDK: ${{ inputs.sdk }} - name: resolve versions id: resolve @@ -120,9 +117,10 @@ runs: id: check-source shell: bash run: | - # Determine which version slots need source checkout. - # A slot needs checkout if it is a head version OR if artifact install failed - # (BUILD_FROM_SOURCE_ was set in the previous step). + # Determine which version slots need source checkout and from which repo. + # A slot needs checkout if it is a head version OR if artifact install failed. + # Platform-source versions may reuse the existing platform-otdfctl-dir + # (when their SHA matches) or need a fresh opentdf/platform checkout. for slot in a b c d; do case "$slot" in a) row=$(echo "${version_info}" | jq -rc '.[0] // empty') ;; @@ -131,41 +129,84 @@ runs: d) row=$(echo "${version_info}" | jq -rc '.[3] // empty') ;; esac if [[ -z "$row" ]]; then - echo "needs-source-${slot}=false" >> "$GITHUB_OUTPUT" + echo "needs-checkout-${slot}=false" >> "$GITHUB_OUTPUT" + echo "is-platform-${slot}=false" >> "$GITHUB_OUTPUT" + echo "use-existing-platform-dir-${slot}=false" >> "$GITHUB_OUTPUT" + echo "checkout-repo-${slot}=" >> "$GITHUB_OUTPUT" + echo "checkout-path-${slot}=" >> "$GITHUB_OUTPUT" continue fi + tag=$(echo "$row" | jq -r '.tag') head=$(echo "$row" | jq -r '.head // false') + sha=$(echo "$row" | jq -r '.sha') + source=$(echo "$row" | jq -r '.source // empty') tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" build_from_source_var="BUILD_FROM_SOURCE_${tag_sanitized}" + needs_source=false if [[ "$head" == "true" || "${!build_from_source_var}" == "true" ]]; then - echo "needs-source-${slot}=true" >> "$GITHUB_OUTPUT" - else - echo "needs-source-${slot}=false" >> "$GITHUB_OUTPUT" + needs_source=true + fi + + is_platform=false + use_existing=false + checkout_repo="$STANDALONE_REPO" + checkout_path="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" + + if [[ "$source" == "platform" ]]; then + # Explicit platform mode: resolver tagged this version as from + # opentdf/platform. Use per-version SHA to decide checkout strategy. + is_platform=true + if [[ "$needs_source" == "true" && -n "$PLATFORM_OTDFCTL_DIR" \ + && -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then + # SHA matches existing platform checkout — reuse via symlink + use_existing=true + needs_source=false + elif [[ "$needs_source" == "true" ]]; then + # Different SHA — need a fresh platform checkout + checkout_repo="$PLATFORM_REPO" + checkout_path="${INPUT_PATH}/${INPUT_SDK}/platform-src/${tag}" + fi + elif [[ "$INPUT_SDK" == "go" && -n "$PLATFORM_OTDFCTL_DIR" && "$needs_source" == "true" ]]; then + # Auto-detect fallback: resolver used standalone repo but the + # test job detected otdfctl in the platform checkout. Symlink all + # head go versions to the existing dir (backward-compatible behavior). + is_platform=true + use_existing=true + needs_source=false fi + + echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT" + echo "is-platform-${slot}=${is_platform}" >> "$GITHUB_OUTPUT" + echo "use-existing-platform-dir-${slot}=${use_existing}" >> "$GITHUB_OUTPUT" + echo "checkout-repo-${slot}=${checkout_repo}" >> "$GITHUB_OUTPUT" + echo "checkout-path-${slot}=${checkout_path}" >> "$GITHUB_OUTPUT" done env: version_info: ${{ inputs.version-info }} + INPUT_PATH: ${{ inputs.path }} + INPUT_SDK: ${{ inputs.sdk }} + PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} + PLATFORM_OTDFCTL_SHA: ${{ inputs.platform-otdfctl-sha }} - - name: symlink platform-embedded otdfctl for head versions - if: env.sdk_source == 'platform' + - name: symlink existing platform checkout shell: bash run: | - # When otdfctl lives inside the platform repo, symlink it into - # sdk/go/src/{tag} so the Makefile builds it the same way. + # For versions that can reuse the already-checked-out platform dir, + # symlink platform-otdfctl-dir into sdk/go/src/{tag}. for slot in a b c d; do case "$slot" in - a) version_json="$VERSION_A" ; needs_source="$NEEDS_SOURCE_A" ;; - b) version_json="$VERSION_B" ; needs_source="$NEEDS_SOURCE_B" ;; - c) version_json="$VERSION_C" ; needs_source="$NEEDS_SOURCE_C" ;; - d) version_json="$VERSION_D" ; needs_source="$NEEDS_SOURCE_D" ;; + a) version_json="$VERSION_A" ; use_existing="$USE_EXISTING_A" ;; + b) version_json="$VERSION_B" ; use_existing="$USE_EXISTING_B" ;; + c) version_json="$VERSION_C" ; use_existing="$USE_EXISTING_C" ;; + d) version_json="$VERSION_D" ; use_existing="$USE_EXISTING_D" ;; esac - if [[ -z "$version_json" || "$needs_source" != "true" ]]; then + if [[ -z "$version_json" || "$use_existing" != "true" ]]; then continue fi tag=$(echo "$version_json" | jq -r '.tag') src_dir="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" - echo "Symlinking platform otdfctl to ${src_dir}" + echo "Symlinking existing platform otdfctl to ${src_dir}" mkdir -p "$(dirname "$src_dir")" ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir" done @@ -177,58 +218,101 @@ runs: VERSION_B: ${{ steps.resolve.outputs.version-b }} VERSION_C: ${{ steps.resolve.outputs.version-c }} VERSION_D: ${{ steps.resolve.outputs.version-d }} - NEEDS_SOURCE_A: ${{ steps.check-source.outputs.needs-source-a }} - NEEDS_SOURCE_B: ${{ steps.check-source.outputs.needs-source-b }} - NEEDS_SOURCE_C: ${{ steps.check-source.outputs.needs-source-c }} - NEEDS_SOURCE_D: ${{ steps.check-source.outputs.needs-source-d }} + USE_EXISTING_A: ${{ steps.check-source.outputs.use-existing-platform-dir-a }} + USE_EXISTING_B: ${{ steps.check-source.outputs.use-existing-platform-dir-b }} + USE_EXISTING_C: ${{ steps.check-source.outputs.use-existing-platform-dir-c }} + USE_EXISTING_D: ${{ steps.check-source.outputs.use-existing-platform-dir-d }} - name: checkout version a uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - env.sdk_source != 'platform' - && steps.resolve.outputs.version-a != '' - && steps.check-source.outputs.needs-source-a == 'true' + steps.resolve.outputs.version-a != '' + && steps.check-source.outputs.needs-checkout-a == 'true' with: - path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-a).tag }} + path: ${{ steps.check-source.outputs.checkout-path-a }} persist-credentials: false ref: ${{ fromJson(steps.resolve.outputs.version-a).sha }} - repository: ${{ env.sdk_repo }} + repository: ${{ steps.check-source.outputs.checkout-repo-a }} - name: checkout version b uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - env.sdk_source != 'platform' - && steps.resolve.outputs.version-b != '' - && steps.check-source.outputs.needs-source-b == 'true' + steps.resolve.outputs.version-b != '' + && steps.check-source.outputs.needs-checkout-b == 'true' with: - path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-b).tag }} + path: ${{ steps.check-source.outputs.checkout-path-b }} persist-credentials: false ref: ${{ fromJson(steps.resolve.outputs.version-b).sha }} - repository: ${{ env.sdk_repo }} + repository: ${{ steps.check-source.outputs.checkout-repo-b }} - name: checkout version c uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - env.sdk_source != 'platform' - && steps.resolve.outputs.version-c != '' - && steps.check-source.outputs.needs-source-c == 'true' + steps.resolve.outputs.version-c != '' + && steps.check-source.outputs.needs-checkout-c == 'true' with: - path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-c).tag }} + path: ${{ steps.check-source.outputs.checkout-path-c }} persist-credentials: false ref: ${{ fromJson(steps.resolve.outputs.version-c).sha }} - repository: ${{ env.sdk_repo }} + repository: ${{ steps.check-source.outputs.checkout-repo-c }} - name: checkout version d uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - env.sdk_source != 'platform' - && steps.resolve.outputs.version-d != '' - && steps.check-source.outputs.needs-source-d == 'true' + steps.resolve.outputs.version-d != '' + && steps.check-source.outputs.needs-checkout-d == 'true' with: - path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-d).tag }} + path: ${{ steps.check-source.outputs.checkout-path-d }} persist-credentials: false ref: ${{ fromJson(steps.resolve.outputs.version-d).sha }} - repository: ${{ env.sdk_repo }} + repository: ${{ steps.check-source.outputs.checkout-repo-d }} + + - name: symlink freshly checked-out platform sources + shell: bash + run: | + # For platform-source versions that were checked out (not reusing the + # existing dir), symlink {platform-src}/{tag}/otdfctl → src/{tag} so + # the Makefile discovers them. + for slot in a b c d; do + case "$slot" in + a) is_platform="$IS_PLATFORM_A" ; needs_checkout="$NEEDS_CHECKOUT_A" + checkout_path="$CHECKOUT_PATH_A" ; version_json="$VERSION_A" ;; + b) is_platform="$IS_PLATFORM_B" ; needs_checkout="$NEEDS_CHECKOUT_B" + checkout_path="$CHECKOUT_PATH_B" ; version_json="$VERSION_B" ;; + c) is_platform="$IS_PLATFORM_C" ; needs_checkout="$NEEDS_CHECKOUT_C" + checkout_path="$CHECKOUT_PATH_C" ; version_json="$VERSION_C" ;; + d) is_platform="$IS_PLATFORM_D" ; needs_checkout="$NEEDS_CHECKOUT_D" + checkout_path="$CHECKOUT_PATH_D" ; version_json="$VERSION_D" ;; + esac + if [[ "$is_platform" != "true" || "$needs_checkout" != "true" || -z "$version_json" ]]; then + continue + fi + tag=$(echo "$version_json" | jq -r '.tag') + src_dir="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" + otdfctl_dir="${checkout_path}/otdfctl" + echo "Symlinking freshly checked-out platform otdfctl ${otdfctl_dir} → ${src_dir}" + mkdir -p "$(dirname "$src_dir")" + ln -sfn "$otdfctl_dir" "$src_dir" + done + env: + INPUT_PATH: ${{ inputs.path }} + INPUT_SDK: ${{ inputs.sdk }} + VERSION_A: ${{ steps.resolve.outputs.version-a }} + VERSION_B: ${{ steps.resolve.outputs.version-b }} + VERSION_C: ${{ steps.resolve.outputs.version-c }} + VERSION_D: ${{ steps.resolve.outputs.version-d }} + IS_PLATFORM_A: ${{ steps.check-source.outputs.is-platform-a }} + IS_PLATFORM_B: ${{ steps.check-source.outputs.is-platform-b }} + IS_PLATFORM_C: ${{ steps.check-source.outputs.is-platform-c }} + IS_PLATFORM_D: ${{ steps.check-source.outputs.is-platform-d }} + NEEDS_CHECKOUT_A: ${{ steps.check-source.outputs.needs-checkout-a }} + NEEDS_CHECKOUT_B: ${{ steps.check-source.outputs.needs-checkout-b }} + NEEDS_CHECKOUT_C: ${{ steps.check-source.outputs.needs-checkout-c }} + NEEDS_CHECKOUT_D: ${{ steps.check-source.outputs.needs-checkout-d }} + CHECKOUT_PATH_A: ${{ steps.check-source.outputs.checkout-path-a }} + CHECKOUT_PATH_B: ${{ steps.check-source.outputs.checkout-path-b }} + CHECKOUT_PATH_C: ${{ steps.check-source.outputs.checkout-path-c }} + CHECKOUT_PATH_D: ${{ steps.check-source.outputs.checkout-path-d }} - name: post checkout cleanups if: inputs.sdk == 'java' From 8f783c3c34b612ca51bca1b14003d4560ff51782 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:19:18 -0400 Subject: [PATCH 10/17] fix: address PR review findings for platform otdfctl migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass --source flag through artifact install step so platform-sourced Go versions get the correct module path in .version files - Strip tag infix (e.g., otdfctl/v0.24.0 → v0.24.0) before normalizing in install_go_release to avoid producing invalid version strings - Move step output expansion to env: block in detect-otdfctl to prevent script injection (consistent with 0c1b428) - Narrow except Exception to GitCommandError in registry.py platform tag query so programming bugs propagate instead of being swallowed - Handle StopIteration explicitly in resolve.py with a clear "main branch not found" error message - Validate source parameter in go_module_path() to reject typos - Add symlink target existence checks after ln -sfn in action.yaml - Add ::warning:: annotation when auto-detect fallback skips SHA verification - Fix typo and minor comment/doc improvements Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 4 +++- otdf-sdk-mgr/README.md | 2 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 5 +++++ otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 7 ++++--- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 3 ++- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 5 ++++- xtest/setup-cli-tool/action.yaml | 17 +++++++++++++++-- 7 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 58eec1342..fe37450e8 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -89,6 +89,8 @@ jobs: JAVA_REF: "${{ inputs.java-ref }}" # When explicitly set to 'platform', tells the Go resolver to resolve # against opentdf/platform tags instead of the standalone otdfctl repo. + # Auto-detection of the platform source is handled downstream by the + # detect-otdfctl step and setup-cli-tool action. OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input @@ -310,7 +312,6 @@ jobs: - name: Detect platform-embedded otdfctl id: detect-otdfctl run: | - PLATFORM_DIR="${{ steps.run-platform.outputs.platform-working-dir }}" if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" @@ -330,6 +331,7 @@ jobs: fi env: OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} + PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} ######### CHECKOUT JS CLI ############# - name: Configure js-sdk diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index 818ee4226..4b930eaae 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -56,7 +56,7 @@ otdf-sdk-mgr java-fixup ## How Release Installs Work -- **Go**: Writes a `.version` file containing `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`); `cli.sh`/`otdfctl.sh` use `go run @{version}` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. +- **Go**: Writes a `.version` file containing `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`); `cli.sh`/`otdfctl.sh` use `go run @` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. - **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/` - **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar` diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 1ef2979d2..5d1a4bcf8 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -146,8 +146,13 @@ def go_install_prefix(source: str | None = None) -> str: return GO_INSTALL_PREFIX_STANDALONE +_VALID_GO_SOURCES = {None, "platform"} + + def go_module_path(source: str | None = None) -> str: """Return the Go module path based on source.""" + if source not in _VALID_GO_SOURCES: + raise ValueError(f"Invalid Go source {source!r}; expected one of {_VALID_GO_SOURCES}") if source == "platform": return GO_MODULE_PATH_PLATFORM return GO_MODULE_PATH diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index dc36ad11a..62113e395 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -40,6 +40,9 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) """ go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) + # Strip tag infix (e.g., "otdfctl/v0.24.0" → "v0.24.0") + if "/" in version: + version = version.rsplit("/", 1)[-1] tag = normalize_version(version) module = go_module_path(source) (dist_dir / ".version").write_text(f"{module}@{tag}\n") @@ -142,9 +145,7 @@ def install_java_release(version: str, dist_dir: Path, **_kwargs: object) -> Non } -def install_release( - sdk: str, version: str, dist_name: str | None = None, **kwargs: object -) -> Path: +def install_release(sdk: str, version: str, dist_name: str | None = None, **kwargs: object) -> Path: """Install a released version of an SDK CLI. Args: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index aae9a4eeb..f95f801f6 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -71,6 +71,7 @@ def fetch_text(url: str) -> str: def list_go_versions() -> list[dict[str, Any]]: """List Go SDK versions from git tags in both standalone and platform repos.""" + import git.exc from git import Git repo = Git() @@ -119,7 +120,7 @@ def list_go_versions() -> list[dict[str, Any]]: "install_method": f"{GO_INSTALL_PREFIX_PLATFORM}@{tag}", "stable": is_stable(version), } - except Exception as e: + except git.exc.GitCommandError as e: print(f"Warning: failed to query platform repo for go tags: {e}", file=sys.stderr) results = list(seen.values()) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 8403759eb..e3f264d74 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -147,7 +147,10 @@ def _annotate(result: ResolveResult) -> ResolveResult: repo = Git() if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] - sha, _ = next(tag for tag in all_heads if "refs/heads/main" in tag) + try: + sha, _ = next(tag for tag in all_heads if "refs/heads/main" in tag) + except StopIteration: + return {"sdk": sdk, "alias": version, "err": f"main branch not found in {sdk_url}"} return _annotate( { "sdk": sdk, diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 1aa56d986..f9a0c8313 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -2,7 +2,7 @@ name: configure-sdks description: Check out and build one or more SDK and its CLI tool for use by xtest inputs: path: - description: The path to checkout the the SDK source code to; concatenated with branch or tag name. + description: The path to check out the SDK source code to; concatenated with branch or tag name. sdk: description: The SDK to configure; one of go, java, js version-info: @@ -101,9 +101,12 @@ runs: echo "Installing ${{ inputs.sdk }} $tag from registry (release: $release)" # Sanitize tag for use as an env var name (replace non-alphanumeric/underscore with _) tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" + source=$(echo "$row" | jq -r '.source // empty') + source_flag="" + [[ -n "$source" ]] && source_flag="--source $source" if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \ --sdk "${{ inputs.sdk }}" --version "$release" \ - --dist-name "$tag"; then + --dist-name "$tag" $source_flag; then echo " Warning: Artifact installation failed for ${{ inputs.sdk }} $tag" echo " Will fall back to building from source" echo "BUILD_FROM_SOURCE_${tag_sanitized}=true" >> "$GITHUB_ENV" @@ -171,6 +174,8 @@ runs: # Auto-detect fallback: resolver used standalone repo but the # test job detected otdfctl in the platform checkout. Symlink all # head go versions to the existing dir (backward-compatible behavior). + # Note: no SHA comparison here — all head versions are symlinked regardless of commit. + echo "::warning::Auto-detecting platform source for Go version ${tag} (SHA ${sha}). Platform dir SHA: ${PLATFORM_OTDFCTL_SHA:-unknown}. No SHA match verification performed." is_platform=true use_existing=true needs_source=false @@ -209,6 +214,10 @@ runs: echo "Symlinking existing platform otdfctl to ${src_dir}" mkdir -p "$(dirname "$src_dir")" ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir" + if [ ! -e "$src_dir" ]; then + echo "::error::Symlink target does not exist: $PLATFORM_OTDFCTL_DIR" + exit 1 + fi done env: PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} @@ -293,6 +302,10 @@ runs: echo "Symlinking freshly checked-out platform otdfctl ${otdfctl_dir} → ${src_dir}" mkdir -p "$(dirname "$src_dir")" ln -sfn "$otdfctl_dir" "$src_dir" + if [ ! -e "$src_dir" ]; then + echo "::error::Symlink target does not exist: ${otdfctl_dir} (does the platform repo contain an otdfctl/ directory?)" + exit 1 + fi done env: INPUT_PATH: ${{ inputs.path }} From 1e5fc53ac51f83d9693c53bc226899e04cadbdf3 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:26:08 -0400 Subject: [PATCH 11/17] fix(sdk-mgr): accept "standalone" as valid Go source in go_module_path The workflow input and CLI advertise "standalone" as a valid source, but go_module_path rejected it with a ValueError. Add "standalone" to the valid set so it maps to the default standalone module path, consistent with go_git_url, go_tag_infix, and go_install_prefix. Co-Authored-By: Claude Opus 4.6 (1M context) --- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 5d1a4bcf8..1b71831df 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -146,7 +146,7 @@ def go_install_prefix(source: str | None = None) -> str: return GO_INSTALL_PREFIX_STANDALONE -_VALID_GO_SOURCES = {None, "platform"} +_VALID_GO_SOURCES = {None, "standalone", "platform"} def go_module_path(source: str | None = None) -> str: From 821c02c05142003c5f1695c53dac9131b2903515 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:30:53 -0400 Subject: [PATCH 12/17] docs(xtest): clarify auto mode resolves releases from standalone The otdfctl-source description implied auto detects from the platform checkout, but release resolution runs before that checkout exists. Update the input description and inline comment to accurately reflect the two-phase behavior: releases resolve from standalone, platform detection applies only to head builds. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index fe37450e8..35dc055e1 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -32,7 +32,7 @@ on: required: false type: string default: auto - description: "otdfctl source: 'auto' (detect from platform checkout), 'standalone', or 'platform'" + description: "otdfctl source: 'auto' (standalone for releases, detect platform for head builds), 'standalone', or 'platform'" workflow_call: inputs: platform-ref: @@ -89,8 +89,8 @@ jobs: JAVA_REF: "${{ inputs.java-ref }}" # When explicitly set to 'platform', tells the Go resolver to resolve # against opentdf/platform tags instead of the standalone otdfctl repo. - # Auto-detection of the platform source is handled downstream by the - # detect-otdfctl step and setup-cli-tool action. + # In 'auto' mode, releases resolve from standalone; the detect-otdfctl + # step later checks for platform-embedded otdfctl for head builds only. OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input From 290154484dcc641a74797b4daa8ad2770c8f0550 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:34:13 -0400 Subject: [PATCH 13/17] fix(setup-cli-tool): require SHA match in auto-detect platform fallback In auto mode, setup-cli-tool previously symlinked all head Go versions to the platform checkout without verifying the commit SHA. This meant otdfctl-ref=my-feature could silently test the platform-embedded otdfctl instead of opentdf/otdfctl@my-feature. Now the fallback only reuses the platform checkout when the resolved SHA matches; otherwise it falls through to a standalone checkout. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 4 +++- xtest/setup-cli-tool/action.yaml | 18 +++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 35dc055e1..9f26ba3ac 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -90,7 +90,9 @@ jobs: # When explicitly set to 'platform', tells the Go resolver to resolve # against opentdf/platform tags instead of the standalone otdfctl repo. # In 'auto' mode, releases resolve from standalone; the detect-otdfctl - # step later checks for platform-embedded otdfctl for head builds only. + # step later checks for platform-embedded otdfctl for head builds only, + # and setup-cli-tool reuses the platform checkout only when the resolved + # SHA matches the platform checkout SHA. OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index f9a0c8313..fe4995e4a 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -172,13 +172,17 @@ runs: fi elif [[ "$INPUT_SDK" == "go" && -n "$PLATFORM_OTDFCTL_DIR" && "$needs_source" == "true" ]]; then # Auto-detect fallback: resolver used standalone repo but the - # test job detected otdfctl in the platform checkout. Symlink all - # head go versions to the existing dir (backward-compatible behavior). - # Note: no SHA comparison here — all head versions are symlinked regardless of commit. - echo "::warning::Auto-detecting platform source for Go version ${tag} (SHA ${sha}). Platform dir SHA: ${PLATFORM_OTDFCTL_SHA:-unknown}. No SHA match verification performed." - is_platform=true - use_existing=true - needs_source=false + # test job detected otdfctl in the platform checkout. Only reuse + # the platform dir when the resolved SHA matches; otherwise fall + # through to a standalone checkout so we test the correct code. + if [[ -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then + echo "Auto-detected platform source for Go version ${tag} — SHA ${sha} matches platform checkout." + is_platform=true + use_existing=true + needs_source=false + else + echo "::notice::Go version ${tag} (SHA ${sha}) does not match platform checkout (SHA ${PLATFORM_OTDFCTL_SHA:-unknown}); checking out standalone repo." + fi fi echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT" From 7e99e2d25b3f9f0105e60b3518326a0653844033 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:51:20 -0400 Subject: [PATCH 14/17] fix: harden validation and error handling for platform otdfctl migration Validate PLATFORM_DIR before using it in detect-otdfctl to prevent wrong-repo SHA when the directory is missing. Remove dead cross-repo SHA comparison in auto-detect fallback (standalone and platform repos have different commit histories). Add input validation to all go_* config helpers and the OTDFCTL_SOURCE env var. Treat pre-warm failure as a hard error for the platform module path. Use array pattern for source_args to prevent shell word-splitting. Improve observability with ::warning:: annotations and version-override logging. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 11 +++++--- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py | 9 ++++++- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 25 +++++++++-------- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 14 +++++++--- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 14 ++++++++-- xtest/setup-cli-tool/action.yaml | 27 +++++++++---------- 6 files changed, 62 insertions(+), 38 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 9f26ba3ac..766a4dcf1 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -90,9 +90,10 @@ jobs: # When explicitly set to 'platform', tells the Go resolver to resolve # against opentdf/platform tags instead of the standalone otdfctl repo. # In 'auto' mode, releases resolve from standalone; the detect-otdfctl - # step later checks for platform-embedded otdfctl for head builds only, - # and setup-cli-tool reuses the platform checkout only when the resolved - # SHA matches the platform checkout SHA. + # step later probes the platform checkout for an embedded otdfctl + # directory, and setup-cli-tool acts on this only for versions that need + # a source checkout (head or artifact-install failure), reusing the + # platform checkout only when the resolved SHA matches. OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input @@ -325,6 +326,10 @@ jobs: echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" fi elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then + if [ -z "$PLATFORM_DIR" ] || [ ! -d "$PLATFORM_DIR/otdfctl" ] || [ ! -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "::error::otdfctl-source=platform requested but ${PLATFORM_DIR:-}/otdfctl does not exist or lacks go.mod" + exit 1 + fi echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py index 1d4d6823a..2dcf6e321 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py @@ -113,8 +113,15 @@ def resolve_versions( raise typer.Exit(2) infix = SDK_TAG_INFIXES.get(sdk) - # Allow overriding the Go SDK source (standalone otdfctl repo vs platform monorepo) + # Allow overriding the Go SDK source via OTDFCTL_SOURCE env var + # (standalone otdfctl repo vs platform monorepo) go_source = os.environ.get("OTDFCTL_SOURCE") if sdk == "go" else None + if go_source and go_source not in ("standalone", "platform"): + typer.echo( + f"Warning: unrecognized OTDFCTL_SOURCE={go_source!r}; expected 'platform' or 'standalone'", + err=True, + ) + go_source = None results: list[ResolveResult] = [] shas: set[str] = set() diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 1b71831df..1046a5ef8 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -116,9 +116,17 @@ def get_sdk_dirs() -> dict[str, Path]: } # When resolving go versions from the platform repo, use "otdfctl" infix -# (tags are otdfctl/v0.X.Y in the platform monorepo) +# (tags are otdfctl/vX.Y.Z in the platform monorepo) SDK_TAG_INFIXES_PLATFORM_GO = "otdfctl" +_VALID_GO_SOURCES = {None, "standalone", "platform"} + + +def _validate_go_source(source: str | None) -> None: + """Raise ValueError if source is not a recognised Go source.""" + if source not in _VALID_GO_SOURCES: + raise ValueError(f"Invalid Go source {source!r}; expected one of {_VALID_GO_SOURCES}") + def go_git_url(source: str | None = None) -> str: """Return the git URL for Go SDK resolution based on source. @@ -127,6 +135,7 @@ def go_git_url(source: str | None = None) -> str: source: "platform" to use the platform monorepo, None/"standalone" for the standalone otdfctl repo. """ + _validate_go_source(source) if source == "platform": return SDK_GIT_URLS["platform"] return SDK_GIT_URLS["go"] @@ -134,25 +143,15 @@ def go_git_url(source: str | None = None) -> str: def go_tag_infix(source: str | None = None) -> str | None: """Return the tag infix for Go SDK resolution based on source.""" + _validate_go_source(source) if source == "platform": return SDK_TAG_INFIXES_PLATFORM_GO return None -def go_install_prefix(source: str | None = None) -> str: - """Return the go install/run prefix based on source.""" - if source == "platform": - return GO_INSTALL_PREFIX_PLATFORM - return GO_INSTALL_PREFIX_STANDALONE - - -_VALID_GO_SOURCES = {None, "standalone", "platform"} - - def go_module_path(source: str | None = None) -> str: """Return the Go module path based on source.""" - if source not in _VALID_GO_SOURCES: - raise ValueError(f"Invalid Go source {source!r}; expected one of {_VALID_GO_SOURCES}") + _validate_go_source(source) if source == "platform": return GO_MODULE_PATH_PLATFORM return GO_MODULE_PATH diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 62113e395..b5e668336 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -11,6 +11,7 @@ from pathlib import Path from otdf_sdk_mgr.config import ( + GO_MODULE_PATH_PLATFORM, LTS_VERSIONS, get_sdk_dir, get_sdk_dirs, @@ -36,7 +37,8 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) Args: version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0"). dist_dir: Target distribution directory. - source: "platform" to use the platform monorepo module path, None for standalone. + source: "platform" to use the platform monorepo module path, + None or "standalone" for standalone. """ go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) @@ -56,9 +58,13 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) text=True, ) if result.returncode != 0: - print( - f" Warning: go install pre-warm failed (will retry at runtime): {result.stderr.strip()}" - ) + msg = f"go install pre-warm failed: {result.stderr.strip()}" + if module == GO_MODULE_PATH_PLATFORM: + raise InstallError( + f"{msg}\n" + f"The platform module path {module}@{tag} may not be published yet." + ) + print(f" Warning: {msg} (will retry at runtime)") print(f" Go release {tag} installed to {dist_dir}") diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index f95f801f6..fcd4f78c8 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -112,7 +112,14 @@ def list_go_versions() -> list[dict[str, Any]]: version = tag.removeprefix(f"{infix}/") if not parse_semver(version): continue - # Platform entries take precedence (canonical location post-migration) + # Platform entries take precedence (canonical location post-migration); + # if the same version exists in both repos, the platform entry + # silently overwrites the standalone one. + if version in seen: + print( + f"Note: version {version} found in both standalone and platform repos; using platform source.", + file=sys.stderr, + ) seen[version] = { "sdk": "go", "version": version, @@ -121,7 +128,10 @@ def list_go_versions() -> list[dict[str, Any]]: "stable": is_stable(version), } except git.exc.GitCommandError as e: - print(f"Warning: failed to query platform repo for go tags: {e}", file=sys.stderr) + print( + f"::warning::Failed to query platform repo for go tags: {e}", + file=sys.stderr, + ) results = list(seen.values()) results.sort(key=lambda r: semver_sort_key(r["version"])) diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index fe4995e4a..29747fa4e 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -12,7 +12,8 @@ inputs: description: >- Absolute path to platform's otdfctl/ directory. When set and sdk is "go", head versions whose SHA matches platform-otdfctl-sha are symlinked from - here instead of checked out separately. + here instead of checked out separately. Used in both explicit platform + mode (source: "platform" in resolved version) and auto-detect mode. platform-otdfctl-sha: description: >- SHA of the commit that platform-otdfctl-dir was checked out at. @@ -102,11 +103,11 @@ runs: # Sanitize tag for use as an env var name (replace non-alphanumeric/underscore with _) tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" source=$(echo "$row" | jq -r '.source // empty') - source_flag="" - [[ -n "$source" ]] && source_flag="--source $source" + source_args=() + [[ -n "$source" ]] && source_args=(--source "$source") if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \ --sdk "${{ inputs.sdk }}" --version "$release" \ - --dist-name "$tag" $source_flag; then + --dist-name "$tag" "${source_args[@]}"; then echo " Warning: Artifact installation failed for ${{ inputs.sdk }} $tag" echo " Will fall back to building from source" echo "BUILD_FROM_SOURCE_${tag_sanitized}=true" >> "$GITHUB_ENV" @@ -172,17 +173,13 @@ runs: fi elif [[ "$INPUT_SDK" == "go" && -n "$PLATFORM_OTDFCTL_DIR" && "$needs_source" == "true" ]]; then # Auto-detect fallback: resolver used standalone repo but the - # test job detected otdfctl in the platform checkout. Only reuse - # the platform dir when the resolved SHA matches; otherwise fall - # through to a standalone checkout so we test the correct code. - if [[ -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then - echo "Auto-detected platform source for Go version ${tag} — SHA ${sha} matches platform checkout." - is_platform=true - use_existing=true - needs_source=false - else - echo "::notice::Go version ${tag} (SHA ${sha}) does not match platform checkout (SHA ${PLATFORM_OTDFCTL_SHA:-unknown}); checking out standalone repo." - fi + # test job detected otdfctl in the platform checkout. + # NOTE: SHA comparison across repos is not meaningful (the standalone + # repo and platform repo have different commit histories), so we + # cannot safely reuse the platform checkout here. Fall through to + # a standalone checkout. To use the platform source, set + # otdfctl-source=platform explicitly. + echo "::notice::Go version ${tag} resolved from standalone repo; platform checkout available but cannot auto-reuse (different repo). Set otdfctl-source=platform to use the platform source." fi echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT" From 13385f513457dd23614fe4b7442e0dfdde619cc1 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 12:13:04 -0400 Subject: [PATCH 15/17] fixup ruff format --- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index b5e668336..0822a063e 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -61,8 +61,7 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) msg = f"go install pre-warm failed: {result.stderr.strip()}" if module == GO_MODULE_PATH_PLATFORM: raise InstallError( - f"{msg}\n" - f"The platform module path {module}@{tag} may not be published yet." + f"{msg}\nThe platform module path {module}@{tag} may not be published yet." ) print(f" Warning: {msg} (will retry at runtime)") print(f" Go release {tag} installed to {dist_dir}") From bfe11196652bff3d51dcdebf51fff0adf7cee4f8 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 21:24:38 -0400 Subject: [PATCH 16/17] refactor(xtest): extract composite actions from xtest.yml Replace ~300 lines of inline workflow steps with three reusable composite actions and two new CLI commands, reducing xtest.yml from 770 to 524 lines. New composite actions: - setup-test-environment: consolidates otdfctl detection, platform version lookup, key management check, root key extraction, and multikas support - setup-sdk-clients: wraps setup-cli-tool with SDK-appropriate caching, go.mod/java .env fixups, and make builds (one call per SDK) - setup-kas-instances: starts all 6 KAS instances via otdf-local ci start-kas New CLI commands: - otdf-sdk-mgr go-fixup: bridges client go.mod to server shared modules for head builds (parallels existing java-fixup) - otdf-local ci start-kas: starts KAS instances in CI and emits GITHUB_OUTPUT lines for log file paths Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 355 ++++------------------ otdf-local/src/otdf_local/ci.py | 223 ++++++++++++++ otdf-local/src/otdf_local/cli.py | 3 + otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py | 36 +++ otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py | 95 ++++++ xtest/setup-kas-instances/action.yaml | 86 ++++++ xtest/setup-sdk-clients/action.yaml | 159 ++++++++++ xtest/setup-test-environment/action.yaml | 140 +++++++++ 8 files changed, 797 insertions(+), 300 deletions(-) create mode 100644 otdf-local/src/otdf_local/ci.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py create mode 100644 xtest/setup-kas-instances/action.yaml create mode 100644 xtest/setup-sdk-clients/action.yaml create mode 100644 xtest/setup-test-environment/action.yaml diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 766a4dcf1..3ad813ba4 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -311,226 +311,56 @@ jobs: with: node-version: "22.x" - ######## DETECT PLATFORM-EMBEDDED OTDFCTL ############# - - name: Detect platform-embedded otdfctl - id: detect-otdfctl - run: | - if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then - if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then - echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" - echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" - echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" - echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" - else - echo "otdfctl not found in platform checkout; using standalone repo" - echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" - fi - elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then - if [ -z "$PLATFORM_DIR" ] || [ ! -d "$PLATFORM_DIR/otdfctl" ] || [ ! -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then - echo "::error::otdfctl-source=platform requested but ${PLATFORM_DIR:-}/otdfctl does not exist or lacks go.mod" - exit 1 - fi - echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" - echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" - echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" - else - echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" - fi - env: - OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} - PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} - - ######### CHECKOUT JS CLI ############# - - name: Configure js-sdk - id: configure-js - uses: ./otdftests/xtest/setup-cli-tool + ######## SETUP TEST ENVIRONMENT ############# + - name: Setup test environment + id: test-env + uses: ./otdftests/xtest/setup-test-environment + with: + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + platform-tag: ${{ matrix.platform-tag }} + otdfctl-source-input: ${{ inputs.otdfctl-source }} + + ######## SETUP SDK CLIENTS ############# + - name: Setup JS SDK client + id: setup-js + uses: ./otdftests/xtest/setup-sdk-clients with: - path: otdftests/xtest/sdk sdk: js version-info: "${{ needs.resolve-versions.outputs.js }}" - - - name: Cache npm - if: fromJson(steps.configure-js.outputs.heads)[0] != null - uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + platform-heads: ${{ needs.resolve-versions.outputs.heads }} + platform-tag: ${{ matrix.platform-tag }} + platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }} + + - name: Setup Go SDK client (otdfctl) + id: setup-go + uses: ./otdftests/xtest/setup-sdk-clients with: - path: ~/.npm - key: npm-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/js/src/**/package-lock.json') }} - restore-keys: | - npm-${{ runner.os }}- - - ######## SETUP THE JS CLI ############# - - name: build and setup the web-sdk cli - id: build-web-sdk - if: fromJson(steps.configure-js.outputs.heads)[0] != null - run: | - make - working-directory: otdftests/xtest/sdk/js - - ######## CHECKOUT GO CLI ############# - - name: Configure otdfctl - id: configure-go - uses: ./otdftests/xtest/setup-cli-tool - with: - path: otdftests/xtest/sdk sdk: go version-info: "${{ needs.resolve-versions.outputs.go }}" - platform-otdfctl-dir: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} - platform-otdfctl-sha: ${{ steps.detect-otdfctl.outputs.otdfctl-sha }} - - - name: Cache Go modules - if: fromJson(steps.configure-go.outputs.heads)[0] != null - uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: go-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/go/src/*/go.sum') }} - restore-keys: | - go-${{ runner.os }}- - - - name: Resolve otdfctl heads - id: resolve-otdfctl-heads - if: fromJson(steps.configure-go.outputs.heads)[0] != null - run: |- - echo "OTDFCTL_HEADS=$OTDFCTL_HEADS" >> "$GITHUB_ENV" - env: - OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }} - - - name: Replace otdfctl go.mod packages, but only at head version of platform - if: >- - steps.detect-otdfctl.outputs.otdfctl-source != 'platform' - && fromJson(steps.configure-go.outputs.heads)[0] != null - && env.FOCUS_SDK == 'go' - && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) - env: - PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} - run: |- - echo "Replacing go.mod packages (standalone otdfctl)..." - PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}" - OTDFCTL_DIR_ABS="$(pwd)/otdftests/xtest/sdk/go/src/" - echo "PLATFORM_DIR_ABS: $PLATFORM_DIR_ABS" - echo "OTDFCTL_DIR_ABS: $OTDFCTL_DIR_ABS" - for head in $(echo "${OTDFCTL_HEADS}" | jq -r '.[]'); do - echo "Processing head: $head" - cd "${OTDFCTL_DIR_ABS}/$head" - for m in lib/fixtures lib/ocrypto protocol/go sdk; do - go mod edit -replace "github.com/opentdf/platform/$m=${PLATFORM_DIR_ABS}/$m" - done - go mod tidy - done - - ######## SETUP THE GO CLI ############# - - name: Prepare go cli - if: fromJson(steps.configure-go.outputs.heads)[0] != null - run: |- - make - working-directory: otdftests/xtest/sdk/go - - ####### CHECKOUT JAVA SDK ############## - - - name: Configure java-sdk - id: configure-java - uses: ./otdftests/xtest/setup-cli-tool + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + platform-heads: ${{ needs.resolve-versions.outputs.heads }} + platform-tag: ${{ matrix.platform-tag }} + platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }} + otdfctl-source: ${{ steps.test-env.outputs.otdfctl-source }} + otdfctl-dir: ${{ steps.test-env.outputs.otdfctl-dir }} + otdfctl-sha: ${{ steps.test-env.outputs.otdfctl-sha }} + focus-sdk: ${{ inputs.focus-sdk || 'all' }} + + - name: Setup Java SDK client + id: setup-java + uses: ./otdftests/xtest/setup-sdk-clients with: - path: otdftests/xtest/sdk sdk: java version-info: "${{ needs.resolve-versions.outputs.java }}" - - - name: Cache Maven repository - if: fromJson(steps.configure-java.outputs.heads)[0] != null - uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 - with: - path: ~/.m2/repository - key: maven-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/java/src/**/pom.xml') }} - restore-keys: | - maven-${{ runner.os }}- - - - name: pre-release protocol buffers for java-sdk - if: >- - fromJson(steps.configure-java.outputs.heads)[0] != null - && (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java') - && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) - run: |- - echo "Replacing .env files for java-sdk..." - echo "Platform tag: $platform_tag" - echo "Java version info: $java_version_info" - for row in $(echo "$java_version_info" | jq -c '.[]'); do - TAG=$(echo "$row" | jq -r '.tag') - HEAD=$(echo "$row" | jq -r '.head') - if [[ "$HEAD" == "true" ]]; then - echo "Creating .env file for tag: [$TAG]; pointing to platform ref [$platform_tag]" - echo "PLATFORM_BRANCH=$platform_ref" > "otdftests/xtest/sdk/java/${TAG}.env" - else - echo "Skipping .env file creation for release version [$TAG]" - fi - done - env: - java_version_info: ${{ needs.resolve-versions.outputs.java }} - platform_ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)[matrix.platform-tag] }} - platform_tag: ${{ matrix.platform-tag }} - - ####### SETUP JAVA CLI ############## - - name: Prepare java cli - if: fromJson(steps.configure-java.outputs.heads)[0] != null - run: | - make - working-directory: otdftests/xtest/sdk/java - env: - BUF_INPUT_HTTPS_USERNAME: opentdf-bot - BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} - - ######## Configure test environment ############# - - name: Lookup current platform version - id: platform-version - run: |- - if ! go run ./service version; then - # NOTE: the version command was added in 0.4.37 - echo "Error: Unable to get platform version; defaulting to tag: [$PLATFORM_TAG]" - echo "PLATFORM_VERSION=$PLATFORM_TAG" >> "$GITHUB_ENV" - exit - fi - # Older version commands output version to stderr; newer versions output to stdout - PLATFORM_VERSION=$(go run ./service version 2>&1) - echo "PLATFORM_VERSION=$PLATFORM_VERSION" >> "$GITHUB_ENV" - echo "## Platform version output: [$PLATFORM_VERSION]" - working-directory: ${{ steps.run-platform.outputs.platform-working-dir }} - env: - PLATFORM_TAG: ${{ matrix.platform-tag }} - - - name: Check key management support and prepare root key - id: km-check - run: |- - OT_CONFIG_FILE="$(pwd)/opentdf.yaml" - echo "OT_CONFIG_FILE=$OT_CONFIG_FILE" >> "$GITHUB_ENV" - # Determine if the config declares the key_management field - km_value=$(yq e '.services.kas.preview.key_management' "$OT_CONFIG_FILE" 2>/dev/null || echo "null") - case "$km_value" in - true|false) - echo "KEY_MANAGEMENT_SUPPORTED=true" >> "$GITHUB_ENV" - echo "supported=true" >> "$GITHUB_OUTPUT" - ;; - *) - echo "KEY_MANAGEMENT_SUPPORTED=false" >> "$GITHUB_ENV" - echo "supported=false" >> "$GITHUB_OUTPUT" - ;; - esac - # Prepare a root key for use by additional KAS instances - existing_root_key=$(yq e '.services.kas.root_key' "$OT_CONFIG_FILE" 2>/dev/null || echo "") - if [ -n "$existing_root_key" ] && [ "$existing_root_key" != "null" ]; then - echo "Using existing root key from config" - echo "OT_ROOT_KEY=$existing_root_key" >> "$GITHUB_ENV" - echo "root_key=$existing_root_key" >> "$GITHUB_OUTPUT" - else - echo "Generating a new root key for additional KAS" - gen_root_key=$(openssl rand -hex 32) - echo "OT_ROOT_KEY=$gen_root_key" >> "$GITHUB_ENV" - echo "root_key=$gen_root_key" >> "$GITHUB_OUTPUT" - fi - working-directory: ${{ steps.run-platform.outputs.platform-working-dir }} - - - name: Install test dependencies - run: uv sync - working-directory: otdftests/xtest + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + platform-heads: ${{ needs.resolve-versions.outputs.heads }} + platform-tag: ${{ matrix.platform-tag }} + platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }} + focus-sdk: ${{ inputs.focus-sdk || 'all' }} + pat-opentdf: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} + + ######## VALIDATE HELPERS ############# - name: Validate xtest helper library (tests of the test harness and its utilities) if: ${{ !inputs }} run: |- @@ -579,92 +409,17 @@ jobs: ######## ATTRIBUTE BASED CONFIGURATION ############# - - name: Does platform support multikas? - id: multikas - run: |- - if [[ $PLATFORM_TAG == main ]]; then - echo "Main supports multikas" - echo "supported=true" >> "$GITHUB_OUTPUT" - elif awk -F. '{ if ($1 > 0 || ($1 == 0 && $2 > 4)) exit 0; else exit 1; }' <<< "${PLATFORM_VERSION#v}"; then - echo "Selected version [$PLATFORM_VERSION] supports multikas" - echo "supported=true" >> "$GITHUB_OUTPUT" - else - echo "At tag [$PLATFORM_TAG], [$PLATFORM_VERSION] probably does not support multikas" - echo "supported=false" >> "$GITHUB_OUTPUT" - fi - env: - PLATFORM_TAG: ${{ matrix.platform-tag }} - - - name: Start additional kas - id: kas-alpha - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - kas-name: alpha - kas-port: 8181 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional kas - id: kas-beta - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - kas-name: beta - kas-port: 8282 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional kas - id: kas-gamma - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - kas-name: gamma - kas-port: 8383 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional kas - id: kas-delta - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix + - name: Start KAS instances for ABAC tests + id: kas-instances + if: ${{ steps.test-env.outputs.multikas-supported == 'true' }} + uses: ./otdftests/xtest/setup-kas-instances with: - ec-tdf-enabled: true - kas-port: 8484 - kas-name: delta - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional KM kas (km1) - id: kas-km1 - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - key-management: ${{ steps.km-check.outputs.supported }} - kas-name: km1 - kas-port: 8585 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional KM kas (km2) - id: kas-km2 - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - kas-name: km2 - key-management: ${{ steps.km-check.outputs.supported }} - kas-port: 8686 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + root-key: ${{ steps.test-env.outputs.root-key }} + key-management-supported: ${{ steps.test-env.outputs.key-management-supported }} - name: Run attribute based configuration tests - if: ${{ steps.multikas.outputs.supported == 'true' }} + if: ${{ steps.test-env.outputs.multikas-supported == 'true' }} run: >- uv run pytest -ra @@ -682,12 +437,12 @@ jobs: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" PLATFORM_TAG: ${{ matrix.platform-tag }} PLATFORM_LOG_FILE: "../../${{ steps.run-platform.outputs.platform-log-file }}" - KAS_ALPHA_LOG_FILE: "../../${{ steps.kas-alpha.outputs.log-file }}" - KAS_BETA_LOG_FILE: "../../${{ steps.kas-beta.outputs.log-file }}" - KAS_GAMMA_LOG_FILE: "../../${{ steps.kas-gamma.outputs.log-file }}" - KAS_DELTA_LOG_FILE: "../../${{ steps.kas-delta.outputs.log-file }}" - KAS_KM1_LOG_FILE: "../../${{ steps.kas-km1.outputs.log-file }}" - KAS_KM2_LOG_FILE: "../../${{ steps.kas-km2.outputs.log-file }}" + KAS_ALPHA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-alpha-log-file }} + KAS_BETA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-beta-log-file }} + KAS_GAMMA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-gamma-log-file }} + KAS_DELTA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-delta-log-file }} + KAS_KM1_LOG_FILE: ${{ steps.kas-instances.outputs.kas-km1-log-file }} + KAS_KM2_LOG_FILE: ${{ steps.kas-instances.outputs.kas-km2-log-file }} - name: Upload artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 diff --git a/otdf-local/src/otdf_local/ci.py b/otdf-local/src/otdf_local/ci.py new file mode 100644 index 000000000..bab3b3203 --- /dev/null +++ b/otdf-local/src/otdf_local/ci.py @@ -0,0 +1,223 @@ +"""CI-specific commands for otdf-local. + +These commands adapt the local environment management for GitHub Actions CI, +where the platform is already started by an external action and we only need +to start KAS instances as background processes. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Annotated + +import typer + +from otdf_local.config.ports import Ports +from otdf_local.config.settings import Settings +from otdf_local.health.waits import WaitTimeoutError, wait_for_health +from otdf_local.services import get_kas_manager +from otdf_local.utils.console import ( + print_error, + print_info, + print_success, + print_warning, +) +from otdf_local.utils.yaml import load_yaml, save_yaml, set_nested + +ci_app = typer.Typer( + name="ci", + help="CI-specific commands for GitHub Actions workflows.", + no_args_is_help=True, +) + + +def _emit_github_output(key: str, value: str) -> None: + """Write a key=value pair to $GITHUB_OUTPUT if available, else print to stdout.""" + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a") as f: + f.write(f"{key}={value}\n") + else: + # Fallback for local testing + print(f"{key}={value}", file=sys.stdout) + + +def _prepare_kas_template( + settings: Settings, root_key: str | None, ec_tdf_enabled: bool +) -> None: + """Ensure the KAS template config has the right root key and EC TDF settings. + + In CI, the platform config may have a root_key that differs from what + we want for additional KAS instances. This updates the platform config + in-place so that KASService._generate_config reads the correct root_key. + """ + if root_key: + config = load_yaml(settings.platform_config) + set_nested(config, "services.kas.root_key", root_key) + if ec_tdf_enabled: + set_nested(config, "services.kas.preview.ec_tdf_enabled", True) + save_yaml(settings.platform_config, config) + + +@ci_app.command("start-kas") +def start_kas( + platform_dir: Annotated[ + Path, + typer.Option( + "--platform-dir", + help="Path to the platform checkout (must contain opentdf-kas-mode.yaml)", + envvar="OTDF_LOCAL_PLATFORM_DIR", + ), + ], + root_key: Annotated[ + str | None, + typer.Option( + "--root-key", + help="Root key for KAS instances (overrides platform config value)", + envvar="OT_ROOT_KEY", + ), + ] = None, + ec_tdf_enabled: Annotated[ + bool, + typer.Option( + "--ec-tdf-enabled/--no-ec-tdf", + help="Enable EC TDF support", + ), + ] = True, + key_management: Annotated[ + bool, + typer.Option( + "--key-management/--no-key-management", + help="Enable key management on km1/km2 instances", + ), + ] = False, + log_type: Annotated[ + str, + typer.Option( + "--log-type", + help="Log format type (json, text)", + ), + ] = "json", + health_timeout: Annotated[ + int, + typer.Option( + "--health-timeout", + help="Seconds to wait for each KAS instance to become healthy", + ), + ] = 60, + instances: Annotated[ + str | None, + typer.Option( + "--instances", + help="Comma-separated KAS instance names (default: all)", + ), + ] = None, +) -> None: + """Start KAS instances for CI and emit GitHub Actions outputs. + + Expects the platform to already be running (started by start-up-with-containers). + Starts all 6 KAS instances (alpha, beta, gamma, delta, km1, km2) as background + processes, waits for each to pass health checks, and emits log file paths as + GitHub Actions step outputs. + + Output keys (written to $GITHUB_OUTPUT): + kas-alpha-log-file, kas-beta-log-file, kas-gamma-log-file, + kas-delta-log-file, kas-km1-log-file, kas-km2-log-file + """ + platform_dir = platform_dir.resolve() + if not platform_dir.is_dir(): + print_error(f"Platform directory does not exist: {platform_dir}") + raise typer.Exit(1) + + # Check for required template files + kas_template = platform_dir / "opentdf-kas-mode.yaml" + platform_config = platform_dir / "opentdf-dev.yaml" + if not kas_template.exists(): + # Fall back to opentdf.yaml if opentdf-kas-mode.yaml doesn't exist + kas_template_alt = platform_dir / "opentdf.yaml" + if kas_template_alt.exists(): + print_info( + f"Using {kas_template_alt} as KAS template (opentdf-kas-mode.yaml not found)" + ) + else: + print_error( + f"Neither opentdf-kas-mode.yaml nor opentdf.yaml found in {platform_dir}" + ) + raise typer.Exit(1) + + if not platform_config.exists(): + # Try opentdf.yaml as fallback + platform_config_alt = platform_dir / "opentdf.yaml" + if platform_config_alt.exists(): + platform_config = platform_config_alt + + # Build settings with CI-specific overrides + # We use a fresh xtest_root derived from this package's location + settings = Settings( + platform_dir=platform_dir, + ) + settings.ensure_directories() + + # Update root key in platform config if provided + if root_key: + _prepare_kas_template(settings, root_key, ec_tdf_enabled) + + # Determine which instances to start + if instances: + kas_names = [n.strip() for n in instances.split(",")] + for name in kas_names: + if name not in Ports.all_kas_names(): + print_error(f"Unknown KAS instance: {name}") + raise typer.Exit(1) + else: + kas_names = Ports.all_kas_names() + + # Start KAS instances + print_info(f"Starting KAS instances: {', '.join(kas_names)}...") + kas_manager = get_kas_manager(settings) + + failed = [] + for name in kas_names: + kas = kas_manager.get(name) + if kas is None: + print_error(f"KAS instance {name} not found in manager") + failed.append(name) + continue + if not kas.start(): + print_error(f"Failed to start KAS {name}") + failed.append(name) + + if failed: + print_error(f"Failed to start: {', '.join(failed)}") + raise typer.Exit(1) + + # Wait for health + print_info("Waiting for KAS health checks...") + unhealthy = [] + for name in kas_names: + port = Ports.get_kas_port(name) + try: + wait_for_health( + f"http://localhost:{port}/healthz", + timeout=health_timeout, + service_name=f"KAS {name}", + ) + except WaitTimeoutError as e: + print_warning(str(e)) + unhealthy.append(name) + + if unhealthy: + print_error(f"KAS instances failed health check: {', '.join(unhealthy)}") + raise typer.Exit(1) + + print_success(f"All {len(kas_names)} KAS instances are healthy") + + # Emit outputs + for name in kas_names: + log_path = settings.get_kas_log_path(name) + output_key = f"kas-{name}-log-file" + _emit_github_output(output_key, str(log_path)) + + print_success("CI KAS startup complete") diff --git a/otdf-local/src/otdf_local/cli.py b/otdf-local/src/otdf_local/cli.py index d8e3597ff..59fc8e098 100644 --- a/otdf-local/src/otdf_local/cli.py +++ b/otdf-local/src/otdf_local/cli.py @@ -11,6 +11,7 @@ from rich.live import Live from otdf_local import __version__ +from otdf_local.ci import ci_app from otdf_local.config.ports import Ports from otdf_local.config.settings import get_settings from otdf_local.health.waits import WaitTimeoutError, wait_for_health, wait_for_port @@ -43,6 +44,8 @@ pretty_exceptions_enable=sys.stderr.isatty(), ) +app.add_typer(ci_app, name="ci") + def _show_provision_error(result: ProvisionResult, target: str) -> None: """Display provisioning error with stderr details.""" diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py index 24148bdd7..62580ebc7 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py @@ -93,3 +93,39 @@ def java_fixup( from otdf_sdk_mgr.java_fixup import post_checkout_java_fixup post_checkout_java_fixup(base_dir) + + +@app.command("go-fixup") +def go_fixup_cmd( + platform_dir: Annotated[ + Path, + typer.Option("--platform-dir", help="Path to the platform checkout root"), + ], + heads: Annotated[ + Optional[str], + typer.Option( + "--heads", + help="JSON list of head version tags to process (e.g. '[\"main\"]')", + ), + ] = None, + base_dir: Annotated[ + Optional[Path], + typer.Argument(help="Base directory for Go source trees"), + ] = None, +) -> None: + """Bridge Go client go.mod to server shared modules for head builds. + + Performs go mod edit -replace + go mod tidy for each head version, + pointing platform module imports at the local platform checkout. + Only needed for standalone otdfctl checkouts. + """ + import json as json_mod + + from otdf_sdk_mgr.go_fixup import go_fixup + + heads_list = json_mod.loads(heads) if heads else None + try: + go_fixup(platform_dir, heads=heads_list, base_dir=base_dir) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) from e diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py new file mode 100644 index 000000000..4098361c2 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py @@ -0,0 +1,95 @@ +"""Post-checkout fixups for Go SDK (otdfctl) source trees. + +Bridges client go.mod to server shared modules for head builds where +client and server share unreleased code. Only applies to standalone +otdfctl checkouts — platform-source builds already have the modules. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from otdf_sdk_mgr.config import get_sdk_dir + +# Platform modules that standalone otdfctl imports and that may need +# a local replace directive when testing against a head platform build. +PLATFORM_MODULES = [ + "lib/fixtures", + "lib/ocrypto", + "protocol/go", + "sdk", +] + + +def go_fixup( + platform_dir: Path, + heads: list[str] | None = None, + base_dir: Path | None = None, +) -> None: + """Replace go.mod references to point at local platform checkout. + + Args: + platform_dir: Absolute path to the platform checkout root + (containing lib/, protocol/, sdk/). + heads: JSON-decoded list of head version tags to process. + If None, all subdirectories under *base_dir* are processed. + base_dir: Directory containing per-version otdfctl source trees + (e.g. ``xtest/sdk/go/src``). Defaults to ``get_sdk_dir() / "go" / "src"``. + """ + if base_dir is None: + base_dir = get_sdk_dir() / "go" / "src" + + if not base_dir.exists(): + print(f"Base directory {base_dir} does not exist, nothing to fix.") + return + + platform_dir = platform_dir.resolve() + if not platform_dir.is_dir(): + raise FileNotFoundError(f"Platform directory does not exist: {platform_dir}") + + dirs_to_process: list[Path] = [] + if heads: + for tag in heads: + d = base_dir / tag + if d.is_dir(): + dirs_to_process.append(d) + else: + print(f"Warning: head directory {d} does not exist, skipping.") + else: + for d in sorted(base_dir.iterdir()): + if d.is_dir() and not d.name.endswith(".git"): + dirs_to_process.append(d) + + if not dirs_to_process: + print("No directories to process.") + return + + for src_dir in dirs_to_process: + if not (src_dir / "go.mod").exists(): + print(f"No go.mod in {src_dir}, skipping.") + continue + + print(f"Applying go.mod replacements in {src_dir}...") + for module in PLATFORM_MODULES: + local_path = platform_dir / module + if not local_path.is_dir(): + print(f" Warning: {local_path} does not exist, skipping {module}") + continue + subprocess.run( + [ + "go", + "mod", + "edit", + "-replace", + f"github.com/opentdf/platform/{module}={local_path}", + ], + cwd=src_dir, + check=True, + ) + print(f" Replaced github.com/opentdf/platform/{module} -> {local_path}") + + print(f"Running go mod tidy in {src_dir}...") + subprocess.run(["go", "mod", "tidy"], cwd=src_dir, check=True) + + print("Go fixup complete.") diff --git a/xtest/setup-kas-instances/action.yaml b/xtest/setup-kas-instances/action.yaml new file mode 100644 index 000000000..f5e420172 --- /dev/null +++ b/xtest/setup-kas-instances/action.yaml @@ -0,0 +1,86 @@ +name: setup-kas-instances +description: >- + Start additional KAS instances for multi-KAS / ABAC tests. + Uses otdf-local ci start-kas to start all 6 KAS instances + (alpha, beta, gamma, delta, km1, km2) and expose their log file paths. + +inputs: + platform-working-dir: + description: Path to the platform checkout directory + required: true + root-key: + description: Root key for KAS instances + required: true + key-management-supported: + description: Enable key management on km1/km2 instances (true/false) + required: false + default: "false" + ec-tdf-enabled: + description: Enable EC TDF support + required: false + default: "true" + log-type: + description: Log format type + required: false + default: "json" + tests-path: + description: Path to the tests repo checkout + required: false + default: "otdftests" + +outputs: + kas-alpha-log-file: + description: Path to KAS alpha log file + value: ${{ steps.start-kas.outputs.kas-alpha-log-file }} + kas-beta-log-file: + description: Path to KAS beta log file + value: ${{ steps.start-kas.outputs.kas-beta-log-file }} + kas-gamma-log-file: + description: Path to KAS gamma log file + value: ${{ steps.start-kas.outputs.kas-gamma-log-file }} + kas-delta-log-file: + description: Path to KAS delta log file + value: ${{ steps.start-kas.outputs.kas-delta-log-file }} + kas-km1-log-file: + description: Path to KAS km1 log file + value: ${{ steps.start-kas.outputs.kas-km1-log-file }} + kas-km2-log-file: + description: Path to KAS km2 log file + value: ${{ steps.start-kas.outputs.kas-km2-log-file }} + +runs: + using: composite + steps: + - name: Start KAS instances + id: start-kas + shell: bash + run: | + KM_FLAG="" + if [[ "$KEY_MANAGEMENT" == "true" ]]; then + KM_FLAG="--key-management" + else + KM_FLAG="--no-key-management" + fi + + EC_FLAG="" + if [[ "$EC_TDF_ENABLED" == "true" ]]; then + EC_FLAG="--ec-tdf-enabled" + else + EC_FLAG="--no-ec-tdf" + fi + + OTDF_LOCAL_DIR="$(cd "$TESTS_PATH" && pwd)/otdf-local" + + uv run --project "$OTDF_LOCAL_DIR" otdf-local ci start-kas \ + --platform-dir "$(pwd)/$PLATFORM_DIR" \ + --root-key "$ROOT_KEY" \ + $EC_FLAG \ + $KM_FLAG \ + --log-type "$LOG_TYPE" + env: + PLATFORM_DIR: ${{ inputs.platform-working-dir }} + ROOT_KEY: ${{ inputs.root-key }} + KEY_MANAGEMENT: ${{ inputs.key-management-supported }} + EC_TDF_ENABLED: ${{ inputs.ec-tdf-enabled }} + LOG_TYPE: ${{ inputs.log-type }} + TESTS_PATH: ${{ inputs.tests-path }} diff --git a/xtest/setup-sdk-clients/action.yaml b/xtest/setup-sdk-clients/action.yaml new file mode 100644 index 000000000..35be35b41 --- /dev/null +++ b/xtest/setup-sdk-clients/action.yaml @@ -0,0 +1,159 @@ +name: setup-sdk-clients +description: >- + Configure, cache, patch, and build an SDK CLI for xtest. Wraps setup-cli-tool + and adds SDK-appropriate caching, go.mod/java .env fixups, and make builds. + Each invocation handles one SDK (go, java, or js). + +inputs: + sdk: + description: "SDK to set up: go, java, or js" + required: true + version-info: + description: JSON-encoded output of otdf-sdk-mgr versions resolve for this SDK + required: true + tests-path: + description: Path to the tests repo checkout + required: false + default: "otdftests" + platform-working-dir: + description: >- + Platform checkout directory. Used for go-fixup (bridging client go.mod + to server shared modules) and detecting platform-embedded otdfctl. + required: false + platform-heads: + description: JSON list of platform tags that are heads (from resolve-versions) + required: false + default: "[]" + platform-tag: + description: Current matrix platform-tag value + required: false + platform-tag-to-sha: + description: JSON object mapping platform tags to SHAs + required: false + default: "{}" + otdfctl-source: + description: "Resolved otdfctl source: platform or standalone" + required: false + default: "standalone" + otdfctl-dir: + description: Absolute path to platform's otdfctl directory + required: false + otdfctl-sha: + description: SHA of the platform otdfctl checkout + required: false + focus-sdk: + description: "SDK focus filter: all, go, java, or js" + required: false + default: "all" + buf-token: + description: BUF token for Java proto compilation + required: false + pat-opentdf: + description: PAT for buf HTTPS password (Java SDK build) + required: false + +outputs: + heads: + description: JSON list of head tags for this SDK + value: ${{ steps.configure.outputs.heads }} + +runs: + using: composite + steps: + # Step 1: Configure SDK via setup-cli-tool (checkout/install) + - name: Configure ${{ inputs.sdk }} + id: configure + uses: ./otdftests/xtest/setup-cli-tool + with: + path: ${{ inputs.tests-path }}/xtest/sdk + sdk: ${{ inputs.sdk }} + version-info: ${{ inputs.version-info }} + platform-otdfctl-dir: ${{ inputs.otdfctl-dir }} + platform-otdfctl-sha: ${{ inputs.otdfctl-sha }} + + # Step 2: SDK-appropriate dependency caching + - name: Cache npm + if: inputs.sdk == 'js' && fromJson(steps.configure.outputs.heads)[0] != null + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/js/src/**/package-lock.json', inputs.tests-path)) }} + restore-keys: | + npm-${{ runner.os }}- + + - name: Cache Go modules + if: inputs.sdk == 'go' && fromJson(steps.configure.outputs.heads)[0] != null + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/go/src/*/go.sum', inputs.tests-path)) }} + restore-keys: | + go-${{ runner.os }}- + + - name: Cache Maven repository + if: inputs.sdk == 'java' && fromJson(steps.configure.outputs.heads)[0] != null + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/java/src/**/pom.xml', inputs.tests-path)) }} + restore-keys: | + maven-${{ runner.os }}- + + # Step 3: SDK-specific fixups + + # Go: Bridge client go.mod to server shared modules (standalone otdfctl only) + - name: Go fixup - replace go.mod packages + if: >- + inputs.sdk == 'go' + && steps.configure.outputs.heads != '[]' + && inputs.otdfctl-source != 'platform' + && inputs.focus-sdk == 'go' + && contains(fromJSON(inputs.platform-heads), inputs.platform-tag) + && inputs.platform-working-dir != '' + shell: bash + run: | + SDK_MGR_DIR="$(cd "$TESTS_PATH" && pwd)/otdf-sdk-mgr" + PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}" + BASE_DIR="$(pwd)/${TESTS_PATH}/xtest/sdk/go/src" + HEADS='${{ steps.configure.outputs.heads }}' + uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr go-fixup \ + --platform-dir "$PLATFORM_DIR_ABS" \ + --heads "$HEADS" \ + "$BASE_DIR" + env: + PLATFORM_WORKING_DIR: ${{ inputs.platform-working-dir }} + TESTS_PATH: ${{ inputs.tests-path }} + + # Java: Create .env files with PLATFORM_BRANCH for head versions + - name: Java fixup - create platform branch .env files + if: >- + inputs.sdk == 'java' + && steps.configure.outputs.heads != '[]' + && (inputs.focus-sdk == 'go' || inputs.focus-sdk == 'java') + && contains(fromJSON(inputs.platform-heads), inputs.platform-tag) + shell: bash + run: | + for row in $(echo "$VERSION_INFO" | jq -c '.[]'); do + TAG=$(echo "$row" | jq -r '.tag') + HEAD=$(echo "$row" | jq -r '.head') + if [[ "$HEAD" == "true" ]]; then + echo "Creating .env file for tag: [$TAG]; pointing to platform ref [$PLATFORM_REF]" + echo "PLATFORM_BRANCH=$PLATFORM_REF" > "${TESTS_PATH}/xtest/sdk/java/${TAG}.env" + fi + done + env: + VERSION_INFO: ${{ inputs.version-info }} + PLATFORM_REF: ${{ fromJSON(inputs.platform-tag-to-sha)[inputs.platform-tag] }} + TESTS_PATH: ${{ inputs.tests-path }} + + # Step 4: Build the SDK CLI + - name: Build ${{ inputs.sdk }} CLI + if: fromJson(steps.configure.outputs.heads)[0] != null + shell: bash + run: make + working-directory: ${{ inputs.tests-path }}/xtest/sdk/${{ inputs.sdk }} + env: + BUF_INPUT_HTTPS_USERNAME: ${{ inputs.sdk == 'java' && 'opentdf-bot' || '' }} + BUF_INPUT_HTTPS_PASSWORD: ${{ inputs.sdk == 'java' && inputs.pat-opentdf || '' }} diff --git a/xtest/setup-test-environment/action.yaml b/xtest/setup-test-environment/action.yaml new file mode 100644 index 000000000..0f8ce86f3 --- /dev/null +++ b/xtest/setup-test-environment/action.yaml @@ -0,0 +1,140 @@ +name: setup-test-environment +description: >- + Detect platform capabilities, extract configuration, and prepare the test + environment. Consolidates otdfctl detection, platform version lookup, key + management support, root key extraction, multikas support check, and test + dependency installation. + +inputs: + platform-working-dir: + description: Platform checkout directory (from start-up-with-containers output) + required: true + platform-tag: + description: Platform version tag under test (matrix value) + required: true + otdfctl-source-input: + description: "User's otdfctl-source preference: auto, standalone, or platform" + required: false + default: "auto" + tests-path: + description: Path to the tests repo checkout + required: false + default: "otdftests" + +outputs: + otdfctl-source: + description: "Resolved otdfctl source: platform or standalone" + value: ${{ steps.detect-otdfctl.outputs.otdfctl-source }} + otdfctl-dir: + description: Absolute path to otdfctl directory (if source=platform) + value: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} + otdfctl-sha: + description: SHA of the platform checkout (if source=platform) + value: ${{ steps.detect-otdfctl.outputs.otdfctl-sha }} + platform-version: + description: Detected platform version string + value: ${{ steps.platform-version.outputs.version }} + key-management-supported: + description: Whether the platform supports key management (true/false) + value: ${{ steps.km-check.outputs.supported }} + root-key: + description: Root key for KAS instances + value: ${{ steps.km-check.outputs.root_key }} + multikas-supported: + description: Whether multi-KAS is supported (true/false) + value: ${{ steps.multikas.outputs.supported }} + +runs: + using: composite + steps: + - name: Detect platform-embedded otdfctl + id: detect-otdfctl + shell: bash + run: | + if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then + if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" + echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" + echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" + else + echo "otdfctl not found in platform checkout; using standalone repo" + echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + fi + elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then + if [ -z "$PLATFORM_DIR" ] || [ ! -d "$PLATFORM_DIR/otdfctl" ] || [ ! -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "::error::otdfctl-source=platform requested but ${PLATFORM_DIR:-}/otdfctl does not exist or lacks go.mod" + exit 1 + fi + echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" + echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" + else + echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + fi + env: + OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source-input }} + PLATFORM_DIR: ${{ inputs.platform-working-dir }} + + - name: Lookup platform version + id: platform-version + shell: bash + run: | + if ! go run ./service version; then + echo "Error: Unable to get platform version; defaulting to tag: [$PLATFORM_TAG]" + echo "version=$PLATFORM_TAG" >> "$GITHUB_OUTPUT" + exit + fi + PLATFORM_VERSION=$(go run ./service version 2>&1) + echo "version=$PLATFORM_VERSION" >> "$GITHUB_OUTPUT" + working-directory: ${{ inputs.platform-working-dir }} + env: + PLATFORM_TAG: ${{ inputs.platform-tag }} + + - name: Check key management support and prepare root key + id: km-check + shell: bash + run: | + OT_CONFIG_FILE="$(pwd)/opentdf.yaml" + km_value=$(yq e '.services.kas.preview.key_management' "$OT_CONFIG_FILE" 2>/dev/null || echo "null") + case "$km_value" in + true|false) + echo "supported=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "supported=false" >> "$GITHUB_OUTPUT" + ;; + esac + existing_root_key=$(yq e '.services.kas.root_key' "$OT_CONFIG_FILE" 2>/dev/null || echo "") + if [ -n "$existing_root_key" ] && [ "$existing_root_key" != "null" ]; then + echo "Using existing root key from config" + echo "root_key=$existing_root_key" >> "$GITHUB_OUTPUT" + else + echo "Generating a new root key for additional KAS" + gen_root_key=$(openssl rand -hex 32) + echo "root_key=$gen_root_key" >> "$GITHUB_OUTPUT" + fi + working-directory: ${{ inputs.platform-working-dir }} + + - name: Check multikas support + id: multikas + shell: bash + run: | + if [[ $PLATFORM_TAG == main ]]; then + echo "Main supports multikas" + echo "supported=true" >> "$GITHUB_OUTPUT" + elif awk -F. '{ if ($1 > 0 || ($1 == 0 && $2 > 4)) exit 0; else exit 1; }' <<< "${PLATFORM_VERSION#v}"; then + echo "Selected version [$PLATFORM_VERSION] supports multikas" + echo "supported=true" >> "$GITHUB_OUTPUT" + else + echo "At tag [$PLATFORM_TAG], [$PLATFORM_VERSION] probably does not support multikas" + echo "supported=false" >> "$GITHUB_OUTPUT" + fi + env: + PLATFORM_TAG: ${{ inputs.platform-tag }} + PLATFORM_VERSION: ${{ steps.platform-version.outputs.version }} + + - name: Install test dependencies + shell: bash + run: uv sync + working-directory: ${{ inputs.tests-path }}/xtest From 9cfc67322f470931c39f4b3b03792d743016c9b0 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 17 Apr 2026 08:04:29 -0400 Subject: [PATCH 17/17] fix: add OT_ROOT_KEY env and guard fromJson on empty outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Add OT_ROOT_KEY to ABAC test step env — the old km-check step set this via $GITHUB_ENV, but setup-test-environment only exposes it as an action output. 2. Guard fromJson() in setup-cli-tool checkout steps with fallback JSON ('{"sha":""}') to prevent JToken errors during Post cleanup when version-b/c/d outputs are empty strings. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 1 + xtest/setup-cli-tool/action.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 3ad813ba4..af47203c8 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -436,6 +436,7 @@ jobs: env: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" PLATFORM_TAG: ${{ matrix.platform-tag }} + OT_ROOT_KEY: ${{ steps.test-env.outputs.root-key }} PLATFORM_LOG_FILE: "../../${{ steps.run-platform.outputs.platform-log-file }}" KAS_ALPHA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-alpha-log-file }} KAS_BETA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-beta-log-file }} diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 29747fa4e..5bf6eff62 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -241,7 +241,7 @@ runs: with: path: ${{ steps.check-source.outputs.checkout-path-a }} persist-credentials: false - ref: ${{ fromJson(steps.resolve.outputs.version-a).sha }} + ref: ${{ fromJson(steps.resolve.outputs.version-a || '{"sha":""}').sha }} repository: ${{ steps.check-source.outputs.checkout-repo-a }} - name: checkout version b @@ -252,7 +252,7 @@ runs: with: path: ${{ steps.check-source.outputs.checkout-path-b }} persist-credentials: false - ref: ${{ fromJson(steps.resolve.outputs.version-b).sha }} + ref: ${{ fromJson(steps.resolve.outputs.version-b || '{"sha":""}').sha }} repository: ${{ steps.check-source.outputs.checkout-repo-b }} - name: checkout version c @@ -263,7 +263,7 @@ runs: with: path: ${{ steps.check-source.outputs.checkout-path-c }} persist-credentials: false - ref: ${{ fromJson(steps.resolve.outputs.version-c).sha }} + ref: ${{ fromJson(steps.resolve.outputs.version-c || '{"sha":""}').sha }} repository: ${{ steps.check-source.outputs.checkout-repo-c }} - name: checkout version d @@ -274,7 +274,7 @@ runs: with: path: ${{ steps.check-source.outputs.checkout-path-d }} persist-credentials: false - ref: ${{ fromJson(steps.resolve.outputs.version-d).sha }} + ref: ${{ fromJson(steps.resolve.outputs.version-d || '{"sha":""}').sha }} repository: ${{ steps.check-source.outputs.checkout-repo-d }} - name: symlink freshly checked-out platform sources