From 3567fd36b7f34c36a6032af835304aaf9abc9f06 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Wed, 15 Apr 2026 16:29:32 -0400 Subject: [PATCH 01/21] feat(xtest): support platform-embedded otdfctl for migration to monorepo otdfctl is moving from opentdf/otdfctl into opentdf/platform. This updates the test infrastructure to auto-detect when the platform checkout contains otdfctl/ and build from there instead of the standalone repo. Key changes: - xtest.yml: new otdfctl-source input (auto/standalone/platform) and detection step that checks for otdfctl/go.mod in the platform dir - setup-cli-tool: new platform-otdfctl-dir input; symlinks platform source into sdk/go/src/ for head builds instead of separate checkout - otdf-sdk-mgr: resolve.py supports go_source="platform" to resolve against the platform repo with otdfctl/ tag infix; installers write .module-path file for the new Go module path - cli.sh/otdfctl.sh: read .module-path to use the correct module path (github.com/opentdf/platform/otdfctl) for go run fallback Backward compatible: old releases still resolve from the standalone repo; .module-path absence defaults to the original module path. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 46 ++++++++++- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py | 6 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 41 ++++++++++ otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 20 +++-- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 79 +++++++++++++------ xtest/sdk/go/cli.sh | 8 +- xtest/sdk/go/otdfctl.sh | 8 +- xtest/setup-cli-tool/action.yaml | 55 +++++++++++-- 8 files changed, 219 insertions(+), 44 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 773448892..e8dc257b0 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -28,6 +28,11 @@ on: type: string default: all description: "SDK to focus on (go, js, java, all)" + otdfctl-source: + required: false + type: string + default: auto + description: "otdfctl source: 'auto' (detect from platform checkout), 'standalone', or 'platform'" workflow_call: inputs: platform-ref: @@ -50,6 +55,10 @@ on: required: false type: string default: all + otdfctl-source: + required: false + type: string + default: auto schedule: - cron: "30 6 * * *" # 0630 UTC - cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday) @@ -198,8 +207,9 @@ jobs: const tagToSha = {}; const headTags = []; - for (const { tag, head, sha, alias, err, release } of refInfo) { - const sdkRepoUrl = `https://github.com/opentdf/${encodeURIComponent(sdkType == 'js' ? 'web-sdk' : sdkType == 'go' ? 'otdfctl' : sdkType == 'java' ? 'java-sdk' : sdkType)}`; + for (const { tag, head, sha, alias, err, release, source } of refInfo) { + const goRepoName = source === 'platform' ? 'platform' : 'otdfctl'; + const sdkRepoUrl = `https://github.com/opentdf/${encodeURIComponent(sdkType == 'js' ? 'web-sdk' : sdkType == 'go' ? goRepoName : sdkType == 'java' ? 'java-sdk' : sdkType)}`; const sdkLink = `${htmlEscape(sdkType)}`; const commitLink = sha ? `${htmlEscape(sha.substring(0, 7))}` : ' . '; const tagLink = (release && tag) @@ -290,6 +300,29 @@ jobs: with: node-version: "22.x" + ######## DETECT PLATFORM-EMBEDDED OTDFCTL ############# + - name: Detect platform-embedded otdfctl + id: detect-otdfctl + run: | + PLATFORM_DIR="${{ steps.run-platform.outputs.platform-working-dir }}" + if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then + if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" + echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" + echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + else + echo "otdfctl not found in platform checkout; using standalone repo" + echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + fi + elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then + echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" + echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + else + echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + fi + env: + OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} + ######### CHECKOUT JS CLI ############# - name: Configure js-sdk id: configure-js @@ -324,6 +357,7 @@ jobs: path: otdftests/xtest/sdk sdk: go version-info: "${{ needs.resolve-versions.outputs.go }}" + platform-otdfctl-dir: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} - name: Cache Go modules if: fromJson(steps.configure-go.outputs.heads)[0] != null @@ -345,11 +379,15 @@ jobs: OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }} - name: Replace otdfctl go.mod packages, but only at head version of platform - if: fromJson(steps.configure-go.outputs.heads)[0] != null && env.FOCUS_SDK == 'go' && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) + if: >- + steps.detect-otdfctl.outputs.otdfctl-source != 'platform' + && fromJson(steps.configure-go.outputs.heads)[0] != null + && env.FOCUS_SDK == 'go' + && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) env: PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} run: |- - echo "Replacing go.mod packages..." + echo "Replacing go.mod packages (standalone otdfctl)..." PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}" OTDFCTL_DIR_ABS="$(pwd)/otdftests/xtest/sdk/go/src/" echo "PLATFORM_DIR_ABS: $PLATFORM_DIR_ABS" diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py index 19188b124..1d4d6823a 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os from typing import Annotated, Any, Optional import typer @@ -112,10 +113,13 @@ def resolve_versions( raise typer.Exit(2) infix = SDK_TAG_INFIXES.get(sdk) + # Allow overriding the Go SDK source (standalone otdfctl repo vs platform monorepo) + go_source = os.environ.get("OTDFCTL_SOURCE") if sdk == "go" else None + results: list[ResolveResult] = [] shas: set[str] = set() for version in tags: - v = resolve(sdk, version, infix) + v = resolve(sdk, version, infix, go_source=go_source) if is_resolve_success(v): env = lookup_additional_options(sdk, v["tag"]) if env: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index adf6c8b1f..25b1b1195 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -71,6 +71,10 @@ def get_sdk_dirs() -> dict[str, Path]: } GO_INSTALL_PREFIX = "go run github.com/opentdf/otdfctl" +GO_INSTALL_PREFIX_PLATFORM = "go run github.com/opentdf/platform/otdfctl" + +GO_MODULE_PATH = "github.com/opentdf/otdfctl" +GO_MODULE_PATH_PLATFORM = "github.com/opentdf/platform/otdfctl" LTS_VERSIONS: dict[str, str] = { "go": "0.24.0", @@ -111,4 +115,41 @@ def get_sdk_dirs() -> dict[str, Path]: "platform": "service", } +# When resolving go versions from the platform repo, use "otdfctl" infix +# (tags are otdfctl/v0.X.Y in the platform monorepo) +SDK_TAG_INFIXES_PLATFORM_GO = "otdfctl" + + +def go_git_url(source: str | None = None) -> str: + """Return the git URL for Go SDK resolution based on source. + + Args: + source: "platform" to use the platform monorepo, None/"standalone" for the + standalone otdfctl repo. + """ + if source == "platform": + return SDK_GIT_URLS["platform"] + return SDK_GIT_URLS["go"] + + +def go_tag_infix(source: str | None = None) -> str | None: + """Return the tag infix for Go SDK resolution based on source.""" + if source == "platform": + return SDK_TAG_INFIXES_PLATFORM_GO + return None + + +def go_install_prefix(source: str | None = None) -> str: + """Return the go install/run prefix based on source.""" + if source == "platform": + return GO_INSTALL_PREFIX_PLATFORM + return GO_INSTALL_PREFIX + + +def go_module_path(source: str | None = None) -> str: + """Return the Go module path based on source.""" + if source == "platform": + return GO_MODULE_PATH_PLATFORM + return GO_MODULE_PATH + ALL_SDKS = ["go", "js", "java"] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index e7c22ae09..395b771fa 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -11,9 +11,11 @@ from pathlib import Path from otdf_sdk_mgr.config import ( + GO_MODULE_PATH_PLATFORM, LTS_VERSIONS, get_sdk_dir, get_sdk_dirs, + go_install_prefix, ) from otdf_sdk_mgr.checkout import checkout_sdk_branch from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions @@ -24,22 +26,30 @@ class InstallError(Exception): """Raised when SDK installation fails.""" -def install_go_release(version: str, dist_dir: Path) -> None: +def install_go_release(version: str, dist_dir: Path, source: str | None = None) -> None: """Install a Go CLI release by writing a .version file. - The cli.sh and otdfctl.sh wrappers read .version and use - `go run github.com/opentdf/otdfctl@{version}` instead of a local binary. + The cli.sh and otdfctl.sh wrappers read .version (and optionally .module-path) + and use `go run @{version}` instead of a local binary. + + Args: + version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0"). + dist_dir: Target distribution directory. + source: "platform" to use the platform monorepo module path, None for standalone. """ go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) tag = normalize_version(version) (dist_dir / ".version").write_text(f"{tag}\n") + if source == "platform": + (dist_dir / ".module-path").write_text(f"{GO_MODULE_PATH_PLATFORM}\n") shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh") shutil.copy(go_dir / "otdfctl.sh", dist_dir / "otdfctl.sh") shutil.copy(go_dir / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml") - print(f" Pre-warming Go cache for otdfctl@{tag}...") + install_module = go_install_prefix(source).removeprefix("go run ") + print(f" Pre-warming Go cache for {install_module}@{tag}...") result = subprocess.run( - ["go", "install", f"github.com/opentdf/otdfctl@{tag}"], + ["go", "install", f"{install_module}@{tag}"], capture_output=True, text=True, ) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 6e4cd7ca1..0c8ad5e9e 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -12,6 +12,8 @@ LTS_VERSIONS, SDK_GIT_URLS, SDK_NPM_PACKAGES, + go_git_url, + go_tag_infix, ) @@ -23,6 +25,7 @@ class ResolveSuccess(TypedDict): pr: NotRequired[str] release: NotRequired[str] sha: str + source: NotRequired[str] tag: str @@ -111,78 +114,104 @@ def lookup_additional_options(sdk: str, version: str) -> str | None: return None -def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: - """Resolve a version spec to a concrete SHA and tag.""" +def resolve( + sdk: str, + version: str, + infix: str | None, + go_source: str | None = None, +) -> ResolveResult: + """Resolve a version spec to a concrete SHA and tag. + + Args: + sdk: SDK identifier (go, js, java, platform). + version: Version spec (main, SHA, tag, latest, lts, etc.). + infix: Tag infix for monorepo tag resolution (e.g. "sdk" for JS). + go_source: For sdk=="go", override the git URL and infix. + "platform" resolves against the platform monorepo (otdfctl/ prefix tags). + None or "standalone" uses the standalone otdfctl repo (default). + """ + _go_platform = sdk == "go" and go_source == "platform" + + def _annotate(result: ResolveResult) -> ResolveResult: + """Add source field to successful results when resolving go from platform.""" + if _go_platform and is_resolve_success(result): + result["source"] = "platform" + return result + try: - sdk_url = SDK_GIT_URLS[sdk] + if _go_platform: + sdk_url = go_git_url("platform") + infix = go_tag_infix("platform") + else: + sdk_url = SDK_GIT_URLS[sdk] repo = Git() if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0] - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "sha": sha, "tag": "main", - } + }) if re.match(SHA_REGEX, version): ls_remote = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] matching_tags = [(sha, tag) for (sha, tag) in ls_remote if sha.startswith(version)] if not matching_tags: - return { + return _annotate({ "sdk": sdk, "alias": version[:7], "sha": version, "tag": version, - } + }) if len(matching_tags) > 1: for sha, tag in matching_tags: if tag.startswith("refs/pull/"): pr_number = tag.split("/")[2] - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "sha": sha, "tag": f"pull-{pr_number}", - } + }) for sha, tag in matching_tags: mq_match = re.match(MERGE_QUEUE_REGEX, tag) if mq_match: to_branch = mq_match.group("branch") pr_number = mq_match.group("pr_number") if to_branch and pr_number: - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "pr": pr_number, "sha": sha, "tag": f"mq-{to_branch}-{pr_number}", - } + }) suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] flattag = "mq--" + suffix.replace("/", "--") - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "sha": sha, "tag": flattag, - } + }) head = False if tag.startswith("refs/heads/"): head = True tag = tag.split("refs/heads/")[-1] flattag = tag.replace("/", "--") - return { + return _annotate({ "sdk": sdk, "alias": version, "head": head, "sha": sha, "tag": flattag, - } + }) return { "sdk": sdk, @@ -197,12 +226,12 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: tag = tag.split("refs/tags/")[-1] if infix: tag = tag.split(f"{infix}/")[-1] - return { + return _annotate({ "sdk": sdk, "alias": version, "sha": sha, "tag": tag, - } + }) if version.startswith("refs/pull/"): merge_heads = [ @@ -216,14 +245,14 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: "err": f"pull request {pr_number} not found in {sdk_url}", } sha, _ = merge_heads[0] - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "pr": pr_number, "sha": sha, "tag": f"pull-{pr_number}", - } + }) remote_tags = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] all_listed_tags = [ @@ -238,13 +267,13 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: if version in all_listed_branches: sha = all_listed_branches[version] - return { + return _annotate({ "sdk": sdk, "alias": version, "head": True, "sha": sha, "tag": version, - } + }) if infix and version.startswith(f"{infix}/"): version = version.split(f"{infix}/")[-1] @@ -288,13 +317,13 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: if not matching_tags: # No versions with CLI found, fall back to building latest from source sha, tag = stable_tags[-1] - return { + return _annotate({ "sdk": sdk, "alias": alias, "head": True, # Mark as head to trigger source checkout "sha": sha, "tag": tag, - } + }) else: matching_tags = stable_tags[-1:] else: @@ -319,13 +348,13 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult: release = tag if infix: release = f"{infix}/{release}" - return { + return _annotate({ "sdk": sdk, "alias": alias, "release": release, "sha": sha, "tag": tag, - } + }) except Exception as e: return { "sdk": sdk, diff --git a/xtest/sdk/go/cli.sh b/xtest/sdk/go/cli.sh index 172aa5b50..18c2ee735 100755 --- a/xtest/sdk/go/cli.sh +++ b/xtest/sdk/go/cli.sh @@ -22,11 +22,15 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then + MODULE_PATH="github.com/opentdf/otdfctl" + if [ -f "$SCRIPT_DIR/.module-path" ]; then + MODULE_PATH=$(tr -d '[:space:]' <"$SCRIPT_DIR/.module-path") + fi if [ -f "$SCRIPT_DIR/.version" ]; then OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") - cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}") + cmd=(go run "${MODULE_PATH}@${OTDFCTL_VERSION}") else - cmd=(go run "github.com/opentdf/otdfctl@latest") + cmd=(go run "${MODULE_PATH}@latest") fi fi diff --git a/xtest/sdk/go/otdfctl.sh b/xtest/sdk/go/otdfctl.sh index 17fbb0c84..0b3a404d0 100755 --- a/xtest/sdk/go/otdfctl.sh +++ b/xtest/sdk/go/otdfctl.sh @@ -17,11 +17,15 @@ source "$XTEST_DIR/test.env" cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then + MODULE_PATH="github.com/opentdf/otdfctl" + if [ -f "$SCRIPT_DIR/.module-path" ]; then + MODULE_PATH=$(tr -d '[:space:]' <"$SCRIPT_DIR/.module-path") + fi if [ -f "$SCRIPT_DIR/.version" ]; then OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") - cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}") + cmd=(go run "${MODULE_PATH}@${OTDFCTL_VERSION}") else - cmd=(go run "github.com/opentdf/otdfctl@latest") + cmd=(go run "${MODULE_PATH}@latest") fi fi diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 9e110ef4f..bee697707 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -8,6 +8,11 @@ inputs: version-info: description: JSON-encoded output of otdf-sdk-mgr versions resolve required: true + platform-otdfctl-dir: + description: >- + Absolute path to platform's otdfctl/ directory. When set and sdk is "go", + head versions are symlinked from here instead of checked out from the + standalone opentdf/otdfctl repo. outputs: version-a: description: "Object containing tag, sha, and name of a version checked out" @@ -33,19 +38,30 @@ runs: run: | case "${{ inputs.sdk }}" in "go") - echo "sdk_repo=opentdf/otdfctl" >> $GITHUB_ENV + if [[ -n "$PLATFORM_OTDFCTL_DIR" && -f "$PLATFORM_OTDFCTL_DIR/go.mod" ]]; then + echo "sdk_repo=opentdf/platform" >> $GITHUB_ENV + echo "sdk_source=platform" >> $GITHUB_ENV + echo "Using platform-embedded otdfctl from $PLATFORM_OTDFCTL_DIR" + else + echo "sdk_repo=opentdf/otdfctl" >> $GITHUB_ENV + echo "sdk_source=standalone" >> $GITHUB_ENV + fi ;; "java") echo "sdk_repo=opentdf/java-sdk" >> $GITHUB_ENV + echo "sdk_source=standalone" >> $GITHUB_ENV ;; "js") echo "sdk_repo=opentdf/web-sdk" >> $GITHUB_ENV + echo "sdk_source=standalone" >> $GITHUB_ENV ;; *) echo "Invalid SDK specified: ${{ inputs.sdk }}" >> $GITHUB_STEP_SUMMARY exit 1 ;; esac + env: + PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} - name: resolve versions id: resolve @@ -131,10 +147,36 @@ runs: env: version_info: ${{ inputs.version-info }} + - name: symlink platform-embedded otdfctl for head versions + if: env.sdk_source == 'platform' + shell: bash + run: | + # When otdfctl lives inside the platform repo, symlink it into + # sdk/go/src/{tag} so the Makefile builds it the same way. + for slot in a b c d; do + case "$slot" in + a) version_json='${{ steps.resolve.outputs.version-a }}' ; needs_source='${{ steps.check-source.outputs.needs-source-a }}' ;; + b) version_json='${{ steps.resolve.outputs.version-b }}' ; needs_source='${{ steps.check-source.outputs.needs-source-b }}' ;; + c) version_json='${{ steps.resolve.outputs.version-c }}' ; needs_source='${{ steps.check-source.outputs.needs-source-c }}' ;; + d) version_json='${{ steps.resolve.outputs.version-d }}' ; needs_source='${{ steps.check-source.outputs.needs-source-d }}' ;; + esac + if [[ -z "$version_json" || "$needs_source" != "true" ]]; then + continue + fi + tag=$(echo "$version_json" | jq -r '.tag') + src_dir="${{ inputs.path }}/${{ inputs.sdk }}/src/${tag}" + echo "Symlinking platform otdfctl to ${src_dir}" + mkdir -p "$(dirname "$src_dir")" + ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir" + done + env: + PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} + - name: checkout version a uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - steps.resolve.outputs.version-a != '' + env.sdk_source != 'platform' + && steps.resolve.outputs.version-a != '' && steps.check-source.outputs.needs-source-a == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-a).tag }} @@ -145,7 +187,8 @@ runs: - name: checkout version b uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - steps.resolve.outputs.version-b != '' + env.sdk_source != 'platform' + && steps.resolve.outputs.version-b != '' && steps.check-source.outputs.needs-source-b == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-b).tag }} @@ -156,7 +199,8 @@ runs: - name: checkout version c uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - steps.resolve.outputs.version-c != '' + env.sdk_source != 'platform' + && steps.resolve.outputs.version-c != '' && steps.check-source.outputs.needs-source-c == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-c).tag }} @@ -167,7 +211,8 @@ runs: - name: checkout version d uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - steps.resolve.outputs.version-d != '' + env.sdk_source != 'platform' + && steps.resolve.outputs.version-d != '' && steps.check-source.outputs.needs-source-d == 'true' with: path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-d).tag }} From 0ea2e5d0290d87bb4b28cf356aa1a744b2041a8b Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 08:27:13 -0400 Subject: [PATCH 02/21] fixup ruff format --- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 1 + otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 172 +++++++++++++---------- 2 files changed, 98 insertions(+), 75 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 25b1b1195..a7416f618 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -152,4 +152,5 @@ def go_module_path(source: str | None = None) -> str: return GO_MODULE_PATH_PLATFORM return GO_MODULE_PATH + ALL_SDKS = ["go", "js", "java"] diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 0c8ad5e9e..a180c60c2 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -148,70 +148,82 @@ def _annotate(result: ResolveResult) -> ResolveResult: if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0] - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": "main", - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": "main", + } + ) if re.match(SHA_REGEX, version): ls_remote = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] matching_tags = [(sha, tag) for (sha, tag) in ls_remote if sha.startswith(version)] if not matching_tags: - return _annotate({ - "sdk": sdk, - "alias": version[:7], - "sha": version, - "tag": version, - }) + return _annotate( + { + "sdk": sdk, + "alias": version[:7], + "sha": version, + "tag": version, + } + ) if len(matching_tags) > 1: for sha, tag in matching_tags: if tag.startswith("refs/pull/"): pr_number = tag.split("/")[2] - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": f"pull-{pr_number}", - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": f"pull-{pr_number}", + } + ) for sha, tag in matching_tags: mq_match = re.match(MERGE_QUEUE_REGEX, tag) if mq_match: to_branch = mq_match.group("branch") pr_number = mq_match.group("pr_number") if to_branch and pr_number: - return _annotate({ + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "pr": pr_number, + "sha": sha, + "tag": f"mq-{to_branch}-{pr_number}", + } + ) + suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] + flattag = "mq--" + suffix.replace("/", "--") + return _annotate( + { "sdk": sdk, "alias": version, "head": True, - "pr": pr_number, "sha": sha, - "tag": f"mq-{to_branch}-{pr_number}", - }) - suffix = tag.split("refs/heads/gh-readonly-queue/")[-1] - flattag = "mq--" + suffix.replace("/", "--") - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": flattag, - }) + "tag": flattag, + } + ) head = False if tag.startswith("refs/heads/"): head = True tag = tag.split("refs/heads/")[-1] flattag = tag.replace("/", "--") - return _annotate({ - "sdk": sdk, - "alias": version, - "head": head, - "sha": sha, - "tag": flattag, - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": head, + "sha": sha, + "tag": flattag, + } + ) return { "sdk": sdk, @@ -226,12 +238,14 @@ def _annotate(result: ResolveResult) -> ResolveResult: tag = tag.split("refs/tags/")[-1] if infix: tag = tag.split(f"{infix}/")[-1] - return _annotate({ - "sdk": sdk, - "alias": version, - "sha": sha, - "tag": tag, - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "sha": sha, + "tag": tag, + } + ) if version.startswith("refs/pull/"): merge_heads = [ @@ -245,14 +259,16 @@ def _annotate(result: ResolveResult) -> ResolveResult: "err": f"pull request {pr_number} not found in {sdk_url}", } sha, _ = merge_heads[0] - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "pr": pr_number, - "sha": sha, - "tag": f"pull-{pr_number}", - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "pr": pr_number, + "sha": sha, + "tag": f"pull-{pr_number}", + } + ) remote_tags = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")] all_listed_tags = [ @@ -267,13 +283,15 @@ def _annotate(result: ResolveResult) -> ResolveResult: if version in all_listed_branches: sha = all_listed_branches[version] - return _annotate({ - "sdk": sdk, - "alias": version, - "head": True, - "sha": sha, - "tag": version, - }) + return _annotate( + { + "sdk": sdk, + "alias": version, + "head": True, + "sha": sha, + "tag": version, + } + ) if infix and version.startswith(f"{infix}/"): version = version.split(f"{infix}/")[-1] @@ -317,13 +335,15 @@ def _annotate(result: ResolveResult) -> ResolveResult: if not matching_tags: # No versions with CLI found, fall back to building latest from source sha, tag = stable_tags[-1] - return _annotate({ - "sdk": sdk, - "alias": alias, - "head": True, # Mark as head to trigger source checkout - "sha": sha, - "tag": tag, - }) + return _annotate( + { + "sdk": sdk, + "alias": alias, + "head": True, # Mark as head to trigger source checkout + "sha": sha, + "tag": tag, + } + ) else: matching_tags = stable_tags[-1:] else: @@ -348,13 +368,15 @@ def _annotate(result: ResolveResult) -> ResolveResult: release = tag if infix: release = f"{infix}/{release}" - return _annotate({ - "sdk": sdk, - "alias": alias, - "release": release, - "sha": sha, - "tag": tag, - }) + return _annotate( + { + "sdk": sdk, + "alias": alias, + "release": release, + "sha": sha, + "tag": tag, + } + ) except Exception as e: return { "sdk": sdk, From 7c20d73a1961c6dced8e2e9ea71bb39d036ffcc4 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 08:45:09 -0400 Subject: [PATCH 03/21] fix(xtest): update remaining otdfctl references for platform monorepo migration - artifactLink in xtest.yml now uses the correct pkg.go.dev module path based on the source field (platform vs standalone) - registry.py list_go_versions() queries both the standalone otdfctl repo and the platform repo (otdfctl/ prefixed tags), deduplicating with platform entries taking precedence - README.md documents both module paths for Go release installs Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 10 +++-- otdf-sdk-mgr/README.md | 2 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 4 +- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 48 ++++++++++++++++++----- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index e8dc257b0..f1b4d1eff 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -179,7 +179,7 @@ jobs: core.summary.addHeading('Versions under Test', 3); - function artifactLink(sdkType, tag, release, head) { + function artifactLink(sdkType, tag, release, head, source) { if (head || !release) return ''; const v = tag.replace(/^v/, ''); if (sdkType === 'js') { @@ -191,7 +191,11 @@ jobs: return `Maven Central`; } if (sdkType === 'go') { - const url = `https://pkg.go.dev/github.com/opentdf/otdfctl@${encodeURIComponent(tag)}`; + const modulePath = source === 'platform' + ? `github.com/opentdf/platform/otdfctl` + : `github.com/opentdf/otdfctl`; + const moduleTag = source === 'platform' ? `otdfctl/${tag}` : tag; + const url = `https://pkg.go.dev/${modulePath}@${encodeURIComponent(moduleTag)}`; return `pkg.go.dev`; } return ''; @@ -215,7 +219,7 @@ jobs: const tagLink = (release && tag) ? `${htmlEscape(tag)}` : tag ? htmlEscape(tag) : 'N/A'; - const artifactCell = artifactLink(sdkType, tag, release, head); + const artifactCell = artifactLink(sdkType, tag, release, head, source); table.push([sdkLink, tagLink, commitLink, alias || 'N/A', artifactCell || 'N/A', err || 'N/A']); if (err) { errorCount += 1; diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index ee2a2a8b7..2d765241d 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -56,7 +56,7 @@ otdf-sdk-mgr java-fixup ## How Release Installs Work -- **Go**: Writes a `.version` file; `cli.sh`/`otdfctl.sh` use `go run github.com/opentdf/otdfctl@{version}` (no local compilation needed, Go caches the binary) +- **Go**: Writes a `.version` file (and optionally `.module-path`); `cli.sh`/`otdfctl.sh` use `go run @{version}` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. - **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/` - **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar` diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index a7416f618..1ef2979d2 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -70,7 +70,7 @@ def get_sdk_dirs() -> dict[str, Path]: "java": "opentdf/java-sdk", } -GO_INSTALL_PREFIX = "go run github.com/opentdf/otdfctl" +GO_INSTALL_PREFIX_STANDALONE = "go run github.com/opentdf/otdfctl" GO_INSTALL_PREFIX_PLATFORM = "go run github.com/opentdf/platform/otdfctl" GO_MODULE_PATH = "github.com/opentdf/otdfctl" @@ -143,7 +143,7 @@ def go_install_prefix(source: str | None = None) -> str: """Return the go install/run prefix based on source.""" if source == "platform": return GO_INSTALL_PREFIX_PLATFORM - return GO_INSTALL_PREFIX + return GO_INSTALL_PREFIX_STANDALONE def go_module_path(source: str | None = None) -> str: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index 8f8dd34e5..aae9a4eeb 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -12,11 +12,13 @@ from typing import Any from otdf_sdk_mgr.config import ( - GO_INSTALL_PREFIX, + GO_INSTALL_PREFIX_PLATFORM, + GO_INSTALL_PREFIX_STANDALONE, SDK_GITHUB_REPOS, SDK_GIT_URLS, SDK_MAVEN_COORDS, SDK_NPM_PACKAGES, + SDK_TAG_INFIXES_PLATFORM_GO, ) from otdf_sdk_mgr.semver import is_stable, parse_semver, semver_sort_key @@ -68,12 +70,14 @@ def fetch_text(url: str) -> str: def list_go_versions() -> list[dict[str, Any]]: - """List Go SDK versions from git tags.""" + """List Go SDK versions from git tags in both standalone and platform repos.""" from git import Git repo = Git() + seen: dict[str, dict[str, Any]] = {} + + # Standalone repo (opentdf/otdfctl): tags like v0.24.0 raw = repo.ls_remote(SDK_GIT_URLS["go"], tags=True) - results = [] for line in raw.strip().split("\n"): if not line: continue @@ -83,16 +87,42 @@ def list_go_versions() -> list[dict[str, Any]]: tag = ref.removeprefix("refs/tags/") if not parse_semver(tag): continue - version = tag - results.append( - { + seen[tag] = { + "sdk": "go", + "version": tag, + "source": "git-tag", + "install_method": f"{GO_INSTALL_PREFIX_STANDALONE}@{tag}", + "stable": is_stable(tag), + } + + # Platform repo (opentdf/platform): tags like otdfctl/v0.X.Y + infix = SDK_TAG_INFIXES_PLATFORM_GO + try: + raw = repo.ls_remote(SDK_GIT_URLS["platform"], tags=True) + for line in raw.strip().split("\n"): + if not line: + continue + _, ref = line.split("\t", 1) + if ref.endswith("^{}"): + continue + tag = ref.removeprefix("refs/tags/") + if not tag.startswith(f"{infix}/"): + continue + version = tag.removeprefix(f"{infix}/") + if not parse_semver(version): + continue + # Platform entries take precedence (canonical location post-migration) + seen[version] = { "sdk": "go", "version": version, - "source": "git-tag", - "install_method": f"{GO_INSTALL_PREFIX}@{version}", + "source": "platform-git-tag", + "install_method": f"{GO_INSTALL_PREFIX_PLATFORM}@{tag}", "stable": is_stable(version), } - ) + except Exception as e: + print(f"Warning: failed to query platform repo for go tags: {e}", file=sys.stderr) + + results = list(seen.values()) results.sort(key=lambda r: semver_sort_key(r["version"])) return results From cd0aec513965bae93d1b42871ece729b3189ac1e Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 08:46:09 -0400 Subject: [PATCH 04/21] fixup pkg.go removes tag prefixes IIRC --- .github/workflows/xtest.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index f1b4d1eff..c1b9b6746 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -194,8 +194,7 @@ jobs: const modulePath = source === 'platform' ? `github.com/opentdf/platform/otdfctl` : `github.com/opentdf/otdfctl`; - const moduleTag = source === 'platform' ? `otdfctl/${tag}` : tag; - const url = `https://pkg.go.dev/${modulePath}@${encodeURIComponent(moduleTag)}`; + const url = `https://pkg.go.dev/${modulePath}@${encodeURIComponent(tag)}`; return `pkg.go.dev`; } return ''; From 1f027001ebe8660b701ef966596841edff80d8d5 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 09:00:41 -0400 Subject: [PATCH 05/21] refactor(xtest): consolidate .version and .module-path into single .version file The Go SDK version config now uses a single .version file with format `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`) instead of separate .version and .module-path files. Shell wrappers fall back to the standalone module path for legacy bare-version files. Co-Authored-By: Claude Opus 4.6 (1M context) --- otdf-sdk-mgr/README.md | 2 +- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 19 +++++++++---------- xtest/sdk/go/cli.sh | 16 +++++++++------- xtest/sdk/go/otdfctl.sh | 16 +++++++++------- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index 2d765241d..818ee4226 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -56,7 +56,7 @@ otdf-sdk-mgr java-fixup ## How Release Installs Work -- **Go**: Writes a `.version` file (and optionally `.module-path`); `cli.sh`/`otdfctl.sh` use `go run @{version}` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. +- **Go**: Writes a `.version` file containing `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`); `cli.sh`/`otdfctl.sh` use `go run @{version}` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. - **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/` - **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar` diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 395b771fa..cb2173800 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -11,11 +11,10 @@ from pathlib import Path from otdf_sdk_mgr.config import ( - GO_MODULE_PATH_PLATFORM, LTS_VERSIONS, get_sdk_dir, get_sdk_dirs, - go_install_prefix, + go_module_path, ) from otdf_sdk_mgr.checkout import checkout_sdk_branch from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions @@ -29,8 +28,10 @@ class InstallError(Exception): def install_go_release(version: str, dist_dir: Path, source: str | None = None) -> None: """Install a Go CLI release by writing a .version file. - The cli.sh and otdfctl.sh wrappers read .version (and optionally .module-path) - and use `go run @{version}` instead of a local binary. + The cli.sh and otdfctl.sh wrappers read .version and use + `go run @{version}` instead of a local binary. + The .version file contains `module-path@version` + (e.g., `github.com/opentdf/otdfctl@v0.24.0`). Args: version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0"). @@ -40,16 +41,14 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) tag = normalize_version(version) - (dist_dir / ".version").write_text(f"{tag}\n") - if source == "platform": - (dist_dir / ".module-path").write_text(f"{GO_MODULE_PATH_PLATFORM}\n") + module = go_module_path(source) + (dist_dir / ".version").write_text(f"{module}@{tag}\n") shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh") shutil.copy(go_dir / "otdfctl.sh", dist_dir / "otdfctl.sh") shutil.copy(go_dir / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml") - install_module = go_install_prefix(source).removeprefix("go run ") - print(f" Pre-warming Go cache for {install_module}@{tag}...") + print(f" Pre-warming Go cache for {module}@{tag}...") result = subprocess.run( - ["go", "install", f"{install_module}@{tag}"], + ["go", "install", f"{module}@{tag}"], capture_output=True, text=True, ) diff --git a/xtest/sdk/go/cli.sh b/xtest/sdk/go/cli.sh index 18c2ee735..f97b20c1a 100755 --- a/xtest/sdk/go/cli.sh +++ b/xtest/sdk/go/cli.sh @@ -22,15 +22,17 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then - MODULE_PATH="github.com/opentdf/otdfctl" - if [ -f "$SCRIPT_DIR/.module-path" ]; then - MODULE_PATH=$(tr -d '[:space:]' <"$SCRIPT_DIR/.module-path") - fi if [ -f "$SCRIPT_DIR/.version" ]; then - OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") - cmd=(go run "${MODULE_PATH}@${OTDFCTL_VERSION}") + VERSION_SPEC=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") + if [[ "$VERSION_SPEC" == *@* ]]; then + # New format: module-path@version + cmd=(go run "$VERSION_SPEC") + else + # Legacy format: bare version tag, default to standalone module + cmd=(go run "github.com/opentdf/otdfctl@${VERSION_SPEC}") + fi else - cmd=(go run "${MODULE_PATH}@latest") + cmd=(go run "github.com/opentdf/otdfctl@latest") fi fi diff --git a/xtest/sdk/go/otdfctl.sh b/xtest/sdk/go/otdfctl.sh index 0b3a404d0..9ba55f054 100755 --- a/xtest/sdk/go/otdfctl.sh +++ b/xtest/sdk/go/otdfctl.sh @@ -17,15 +17,17 @@ source "$XTEST_DIR/test.env" cmd=("$SCRIPT_DIR"/otdfctl) if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then - MODULE_PATH="github.com/opentdf/otdfctl" - if [ -f "$SCRIPT_DIR/.module-path" ]; then - MODULE_PATH=$(tr -d '[:space:]' <"$SCRIPT_DIR/.module-path") - fi if [ -f "$SCRIPT_DIR/.version" ]; then - OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") - cmd=(go run "${MODULE_PATH}@${OTDFCTL_VERSION}") + VERSION_SPEC=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version") + if [[ "$VERSION_SPEC" == *@* ]]; then + # New format: module-path@version + cmd=(go run "$VERSION_SPEC") + else + # Legacy format: bare version tag, default to standalone module + cmd=(go run "github.com/opentdf/otdfctl@${VERSION_SPEC}") + fi else - cmd=(go run "${MODULE_PATH}@latest") + cmd=(go run "github.com/opentdf/otdfctl@latest") fi fi From 9b9c9d78ae3328f5a9b64d1070cf560c24540f4d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 09:13:27 -0400 Subject: [PATCH 06/21] feat(sdk-mgr): wire --source option through install artifact command Allows specifying the source repo for Go CLI installs (e.g., --source platform) to support the otdfctl migration from standalone repo to the platform monorepo. Co-Authored-By: Claude Opus 4.6 (1M context) --- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py | 6 +++++- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py index e3950d717..e62ae2464 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py @@ -74,12 +74,16 @@ def artifact( dist_name: Annotated[ Optional[str], typer.Option("--dist-name", help="Override dist directory name") ] = None, + source: Annotated[ + Optional[str], + typer.Option(help='Source repo for Go CLI (e.g., "platform" for monorepo)'), + ] = None, ) -> None: """Install a single SDK version (used by CI).""" from otdf_sdk_mgr.installers import InstallError, cmd_install try: - cmd_install(sdk, version, dist_name=dist_name) + cmd_install(sdk, version, dist_name=dist_name, source=source) except InstallError as e: typer.echo(f"Error: {e}", err=True) raise typer.Exit(1) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index cb2173800..dc36ad11a 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -59,7 +59,7 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) print(f" Go release {tag} installed to {dist_dir}") -def install_js_release(version: str, dist_dir: Path) -> None: +def install_js_release(version: str, dist_dir: Path, **_kwargs: object) -> None: """Install a JS CLI release from npm registry.""" js_dir = get_sdk_dir() / "js" dist_dir.mkdir(parents=True, exist_ok=True) @@ -74,7 +74,7 @@ def install_js_release(version: str, dist_dir: Path) -> None: print(f" JS release {v} installed to {dist_dir}") -def install_java_release(version: str, dist_dir: Path) -> None: +def install_java_release(version: str, dist_dir: Path, **_kwargs: object) -> None: """Install a Java CLI release by downloading cmdline.jar from GitHub Releases. Raises InstallError if the artifact is not available or download fails, @@ -142,13 +142,17 @@ def install_java_release(version: str, dist_dir: Path) -> None: } -def install_release(sdk: str, version: str, dist_name: str | None = None) -> Path: +def install_release( + sdk: str, version: str, dist_name: str | None = None, **kwargs: object +) -> Path: """Install a released version of an SDK CLI. Args: sdk: One of "go", "js", "java" version: Version string (e.g., "v0.24.0" or "0.24.0") dist_name: Override the dist directory name (defaults to normalized version) + **kwargs: Extra arguments forwarded to the SDK installer + (e.g., source="platform" for Go). Returns: Path to the created dist directory @@ -166,7 +170,7 @@ def install_release(sdk: str, version: str, dist_name: str | None = None) -> Pat print(f" Dist directory already exists: {dist_dir} (skipping)") return dist_dir - INSTALLERS[sdk](version, dist_dir) + INSTALLERS[sdk](version, dist_dir, **kwargs) return dist_dir @@ -233,7 +237,9 @@ def cmd_release(specs: list[str]) -> None: install_release(sdk, version) -def cmd_install(sdk: str, version: str, dist_name: str | None = None) -> None: +def cmd_install( + sdk: str, version: str, dist_name: str | None = None, source: str | None = None +) -> None: """Install a single SDK version (used by CI action).""" print(f"Installing {sdk} {version}...") - install_release(sdk, version, dist_name=dist_name) + install_release(sdk, version, dist_name=dist_name, source=source) From 0c1b428c19ff253aead560d593d869d257c6be82 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 09:18:40 -0400 Subject: [PATCH 07/21] fix(setup-cli-tool): avoid script injection by using env vars for inputs and step outputs Co-Authored-By: Claude Sonnet 4.6 --- xtest/setup-cli-tool/action.yaml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index bee697707..eb992db2e 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -155,22 +155,32 @@ runs: # sdk/go/src/{tag} so the Makefile builds it the same way. for slot in a b c d; do case "$slot" in - a) version_json='${{ steps.resolve.outputs.version-a }}' ; needs_source='${{ steps.check-source.outputs.needs-source-a }}' ;; - b) version_json='${{ steps.resolve.outputs.version-b }}' ; needs_source='${{ steps.check-source.outputs.needs-source-b }}' ;; - c) version_json='${{ steps.resolve.outputs.version-c }}' ; needs_source='${{ steps.check-source.outputs.needs-source-c }}' ;; - d) version_json='${{ steps.resolve.outputs.version-d }}' ; needs_source='${{ steps.check-source.outputs.needs-source-d }}' ;; + a) version_json="$VERSION_A" ; needs_source="$NEEDS_SOURCE_A" ;; + b) version_json="$VERSION_B" ; needs_source="$NEEDS_SOURCE_B" ;; + c) version_json="$VERSION_C" ; needs_source="$NEEDS_SOURCE_C" ;; + d) version_json="$VERSION_D" ; needs_source="$NEEDS_SOURCE_D" ;; esac if [[ -z "$version_json" || "$needs_source" != "true" ]]; then continue fi tag=$(echo "$version_json" | jq -r '.tag') - src_dir="${{ inputs.path }}/${{ inputs.sdk }}/src/${tag}" + src_dir="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" echo "Symlinking platform otdfctl to ${src_dir}" mkdir -p "$(dirname "$src_dir")" ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir" done env: PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} + INPUT_PATH: ${{ inputs.path }} + INPUT_SDK: ${{ inputs.sdk }} + VERSION_A: ${{ steps.resolve.outputs.version-a }} + VERSION_B: ${{ steps.resolve.outputs.version-b }} + VERSION_C: ${{ steps.resolve.outputs.version-c }} + VERSION_D: ${{ steps.resolve.outputs.version-d }} + NEEDS_SOURCE_A: ${{ steps.check-source.outputs.needs-source-a }} + NEEDS_SOURCE_B: ${{ steps.check-source.outputs.needs-source-b }} + NEEDS_SOURCE_C: ${{ steps.check-source.outputs.needs-source-c }} + NEEDS_SOURCE_D: ${{ steps.check-source.outputs.needs-source-d }} - name: checkout version a uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 From bf739dc58bd22ee5ab4db27f53f0f9f35e4fa751 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 09:07:39 -0400 Subject: [PATCH 08/21] Apply suggestion from @gemini-code-assist[bot] Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index a180c60c2..8403759eb 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -147,7 +147,7 @@ def _annotate(result: ResolveResult) -> ResolveResult: repo = Git() if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] - sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0] + sha, _ = next(tag for tag in all_heads if "refs/heads/main" in tag) return _annotate( { "sdk": sdk, From edb5e3a37d48fc3ea129fb1d3ef2007cd743f541 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 10:59:36 -0400 Subject: [PATCH 09/21] feat(setup-cli-tool): support multiple platform-source Go versions Replace global sdk_source/sdk_repo with per-version checkout decisions so that multiple Go head versions from opentdf/platform at different SHAs can be tested against each other (e.g. a platform PR vs main). - Add platform-otdfctl-sha input for SHA-based matching of the existing platform checkout; versions with a different SHA get a fresh checkout of opentdf/platform into platform-src/{tag}/ with a symlink from otdfctl/ into src/{tag}/ - Wire OTDFCTL_SOURCE to the resolve-versions job when otdfctl-source is explicitly 'platform', so the resolver returns platform SHAs - Preserve backward-compatible auto-detect behavior: when platform- otdfctl-dir is set but version-info lacks source:"platform", all head Go versions still symlink to the existing dir Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 6 + xtest/setup-cli-tool/action.yaml | 202 ++++++++++++++++++++++--------- 2 files changed, 149 insertions(+), 59 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index c1b9b6746..58eec1342 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -87,6 +87,9 @@ jobs: JS_REF: "${{ inputs.js-ref }}" OTDFCTL_REF: "${{ inputs.otdfctl-ref }}" JAVA_REF: "${{ inputs.java-ref }}" + # When explicitly set to 'platform', tells the Go resolver to resolve + # against opentdf/platform tags instead of the standalone otdfctl repo. + OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input if: ${{ inputs.focus-sdk != '' }} @@ -313,6 +316,7 @@ jobs: echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" else echo "otdfctl not found in platform checkout; using standalone repo" echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" @@ -320,6 +324,7 @@ jobs: elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" else echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" fi @@ -361,6 +366,7 @@ jobs: sdk: go version-info: "${{ needs.resolve-versions.outputs.go }}" platform-otdfctl-dir: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} + platform-otdfctl-sha: ${{ steps.detect-otdfctl.outputs.otdfctl-sha }} - name: Cache Go modules if: fromJson(steps.configure-go.outputs.heads)[0] != null diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index eb992db2e..1aa56d986 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -11,8 +11,13 @@ inputs: platform-otdfctl-dir: description: >- Absolute path to platform's otdfctl/ directory. When set and sdk is "go", - head versions are symlinked from here instead of checked out from the - standalone opentdf/otdfctl repo. + head versions whose SHA matches platform-otdfctl-sha are symlinked from + here instead of checked out separately. + platform-otdfctl-sha: + description: >- + SHA of the commit that platform-otdfctl-dir was checked out at. + Used to decide which Go head version (if any) can reuse the existing + platform checkout vs needing a fresh one. outputs: version-a: description: "Object containing tag, sha, and name of a version checked out" @@ -33,35 +38,27 @@ outputs: runs: using: composite steps: - - name: identify repo url + - name: identify repo urls shell: bash run: | - case "${{ inputs.sdk }}" in + case "$INPUT_SDK" in "go") - if [[ -n "$PLATFORM_OTDFCTL_DIR" && -f "$PLATFORM_OTDFCTL_DIR/go.mod" ]]; then - echo "sdk_repo=opentdf/platform" >> $GITHUB_ENV - echo "sdk_source=platform" >> $GITHUB_ENV - echo "Using platform-embedded otdfctl from $PLATFORM_OTDFCTL_DIR" - else - echo "sdk_repo=opentdf/otdfctl" >> $GITHUB_ENV - echo "sdk_source=standalone" >> $GITHUB_ENV - fi + echo "STANDALONE_REPO=opentdf/otdfctl" >> "$GITHUB_ENV" ;; "java") - echo "sdk_repo=opentdf/java-sdk" >> $GITHUB_ENV - echo "sdk_source=standalone" >> $GITHUB_ENV + echo "STANDALONE_REPO=opentdf/java-sdk" >> "$GITHUB_ENV" ;; "js") - echo "sdk_repo=opentdf/web-sdk" >> $GITHUB_ENV - echo "sdk_source=standalone" >> $GITHUB_ENV + echo "STANDALONE_REPO=opentdf/web-sdk" >> "$GITHUB_ENV" ;; *) - echo "Invalid SDK specified: ${{ inputs.sdk }}" >> $GITHUB_STEP_SUMMARY + echo "Invalid SDK specified: $INPUT_SDK" >> "$GITHUB_STEP_SUMMARY" exit 1 ;; esac + echo "PLATFORM_REPO=opentdf/platform" >> "$GITHUB_ENV" env: - PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} + INPUT_SDK: ${{ inputs.sdk }} - name: resolve versions id: resolve @@ -120,9 +117,10 @@ runs: id: check-source shell: bash run: | - # Determine which version slots need source checkout. - # A slot needs checkout if it is a head version OR if artifact install failed - # (BUILD_FROM_SOURCE_ was set in the previous step). + # Determine which version slots need source checkout and from which repo. + # A slot needs checkout if it is a head version OR if artifact install failed. + # Platform-source versions may reuse the existing platform-otdfctl-dir + # (when their SHA matches) or need a fresh opentdf/platform checkout. for slot in a b c d; do case "$slot" in a) row=$(echo "${version_info}" | jq -rc '.[0] // empty') ;; @@ -131,41 +129,84 @@ runs: d) row=$(echo "${version_info}" | jq -rc '.[3] // empty') ;; esac if [[ -z "$row" ]]; then - echo "needs-source-${slot}=false" >> "$GITHUB_OUTPUT" + echo "needs-checkout-${slot}=false" >> "$GITHUB_OUTPUT" + echo "is-platform-${slot}=false" >> "$GITHUB_OUTPUT" + echo "use-existing-platform-dir-${slot}=false" >> "$GITHUB_OUTPUT" + echo "checkout-repo-${slot}=" >> "$GITHUB_OUTPUT" + echo "checkout-path-${slot}=" >> "$GITHUB_OUTPUT" continue fi + tag=$(echo "$row" | jq -r '.tag') head=$(echo "$row" | jq -r '.head // false') + sha=$(echo "$row" | jq -r '.sha') + source=$(echo "$row" | jq -r '.source // empty') tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" build_from_source_var="BUILD_FROM_SOURCE_${tag_sanitized}" + needs_source=false if [[ "$head" == "true" || "${!build_from_source_var}" == "true" ]]; then - echo "needs-source-${slot}=true" >> "$GITHUB_OUTPUT" - else - echo "needs-source-${slot}=false" >> "$GITHUB_OUTPUT" + needs_source=true + fi + + is_platform=false + use_existing=false + checkout_repo="$STANDALONE_REPO" + checkout_path="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" + + if [[ "$source" == "platform" ]]; then + # Explicit platform mode: resolver tagged this version as from + # opentdf/platform. Use per-version SHA to decide checkout strategy. + is_platform=true + if [[ "$needs_source" == "true" && -n "$PLATFORM_OTDFCTL_DIR" \ + && -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then + # SHA matches existing platform checkout — reuse via symlink + use_existing=true + needs_source=false + elif [[ "$needs_source" == "true" ]]; then + # Different SHA — need a fresh platform checkout + checkout_repo="$PLATFORM_REPO" + checkout_path="${INPUT_PATH}/${INPUT_SDK}/platform-src/${tag}" + fi + elif [[ "$INPUT_SDK" == "go" && -n "$PLATFORM_OTDFCTL_DIR" && "$needs_source" == "true" ]]; then + # Auto-detect fallback: resolver used standalone repo but the + # test job detected otdfctl in the platform checkout. Symlink all + # head go versions to the existing dir (backward-compatible behavior). + is_platform=true + use_existing=true + needs_source=false fi + + echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT" + echo "is-platform-${slot}=${is_platform}" >> "$GITHUB_OUTPUT" + echo "use-existing-platform-dir-${slot}=${use_existing}" >> "$GITHUB_OUTPUT" + echo "checkout-repo-${slot}=${checkout_repo}" >> "$GITHUB_OUTPUT" + echo "checkout-path-${slot}=${checkout_path}" >> "$GITHUB_OUTPUT" done env: version_info: ${{ inputs.version-info }} + INPUT_PATH: ${{ inputs.path }} + INPUT_SDK: ${{ inputs.sdk }} + PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} + PLATFORM_OTDFCTL_SHA: ${{ inputs.platform-otdfctl-sha }} - - name: symlink platform-embedded otdfctl for head versions - if: env.sdk_source == 'platform' + - name: symlink existing platform checkout shell: bash run: | - # When otdfctl lives inside the platform repo, symlink it into - # sdk/go/src/{tag} so the Makefile builds it the same way. + # For versions that can reuse the already-checked-out platform dir, + # symlink platform-otdfctl-dir into sdk/go/src/{tag}. for slot in a b c d; do case "$slot" in - a) version_json="$VERSION_A" ; needs_source="$NEEDS_SOURCE_A" ;; - b) version_json="$VERSION_B" ; needs_source="$NEEDS_SOURCE_B" ;; - c) version_json="$VERSION_C" ; needs_source="$NEEDS_SOURCE_C" ;; - d) version_json="$VERSION_D" ; needs_source="$NEEDS_SOURCE_D" ;; + a) version_json="$VERSION_A" ; use_existing="$USE_EXISTING_A" ;; + b) version_json="$VERSION_B" ; use_existing="$USE_EXISTING_B" ;; + c) version_json="$VERSION_C" ; use_existing="$USE_EXISTING_C" ;; + d) version_json="$VERSION_D" ; use_existing="$USE_EXISTING_D" ;; esac - if [[ -z "$version_json" || "$needs_source" != "true" ]]; then + if [[ -z "$version_json" || "$use_existing" != "true" ]]; then continue fi tag=$(echo "$version_json" | jq -r '.tag') src_dir="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" - echo "Symlinking platform otdfctl to ${src_dir}" + echo "Symlinking existing platform otdfctl to ${src_dir}" mkdir -p "$(dirname "$src_dir")" ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir" done @@ -177,58 +218,101 @@ runs: VERSION_B: ${{ steps.resolve.outputs.version-b }} VERSION_C: ${{ steps.resolve.outputs.version-c }} VERSION_D: ${{ steps.resolve.outputs.version-d }} - NEEDS_SOURCE_A: ${{ steps.check-source.outputs.needs-source-a }} - NEEDS_SOURCE_B: ${{ steps.check-source.outputs.needs-source-b }} - NEEDS_SOURCE_C: ${{ steps.check-source.outputs.needs-source-c }} - NEEDS_SOURCE_D: ${{ steps.check-source.outputs.needs-source-d }} + USE_EXISTING_A: ${{ steps.check-source.outputs.use-existing-platform-dir-a }} + USE_EXISTING_B: ${{ steps.check-source.outputs.use-existing-platform-dir-b }} + USE_EXISTING_C: ${{ steps.check-source.outputs.use-existing-platform-dir-c }} + USE_EXISTING_D: ${{ steps.check-source.outputs.use-existing-platform-dir-d }} - name: checkout version a uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - env.sdk_source != 'platform' - && steps.resolve.outputs.version-a != '' - && steps.check-source.outputs.needs-source-a == 'true' + steps.resolve.outputs.version-a != '' + && steps.check-source.outputs.needs-checkout-a == 'true' with: - path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-a).tag }} + path: ${{ steps.check-source.outputs.checkout-path-a }} persist-credentials: false ref: ${{ fromJson(steps.resolve.outputs.version-a).sha }} - repository: ${{ env.sdk_repo }} + repository: ${{ steps.check-source.outputs.checkout-repo-a }} - name: checkout version b uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - env.sdk_source != 'platform' - && steps.resolve.outputs.version-b != '' - && steps.check-source.outputs.needs-source-b == 'true' + steps.resolve.outputs.version-b != '' + && steps.check-source.outputs.needs-checkout-b == 'true' with: - path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-b).tag }} + path: ${{ steps.check-source.outputs.checkout-path-b }} persist-credentials: false ref: ${{ fromJson(steps.resolve.outputs.version-b).sha }} - repository: ${{ env.sdk_repo }} + repository: ${{ steps.check-source.outputs.checkout-repo-b }} - name: checkout version c uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - env.sdk_source != 'platform' - && steps.resolve.outputs.version-c != '' - && steps.check-source.outputs.needs-source-c == 'true' + steps.resolve.outputs.version-c != '' + && steps.check-source.outputs.needs-checkout-c == 'true' with: - path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-c).tag }} + path: ${{ steps.check-source.outputs.checkout-path-c }} persist-credentials: false ref: ${{ fromJson(steps.resolve.outputs.version-c).sha }} - repository: ${{ env.sdk_repo }} + repository: ${{ steps.check-source.outputs.checkout-repo-c }} - name: checkout version d uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: >- - env.sdk_source != 'platform' - && steps.resolve.outputs.version-d != '' - && steps.check-source.outputs.needs-source-d == 'true' + steps.resolve.outputs.version-d != '' + && steps.check-source.outputs.needs-checkout-d == 'true' with: - path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-d).tag }} + path: ${{ steps.check-source.outputs.checkout-path-d }} persist-credentials: false ref: ${{ fromJson(steps.resolve.outputs.version-d).sha }} - repository: ${{ env.sdk_repo }} + repository: ${{ steps.check-source.outputs.checkout-repo-d }} + + - name: symlink freshly checked-out platform sources + shell: bash + run: | + # For platform-source versions that were checked out (not reusing the + # existing dir), symlink {platform-src}/{tag}/otdfctl → src/{tag} so + # the Makefile discovers them. + for slot in a b c d; do + case "$slot" in + a) is_platform="$IS_PLATFORM_A" ; needs_checkout="$NEEDS_CHECKOUT_A" + checkout_path="$CHECKOUT_PATH_A" ; version_json="$VERSION_A" ;; + b) is_platform="$IS_PLATFORM_B" ; needs_checkout="$NEEDS_CHECKOUT_B" + checkout_path="$CHECKOUT_PATH_B" ; version_json="$VERSION_B" ;; + c) is_platform="$IS_PLATFORM_C" ; needs_checkout="$NEEDS_CHECKOUT_C" + checkout_path="$CHECKOUT_PATH_C" ; version_json="$VERSION_C" ;; + d) is_platform="$IS_PLATFORM_D" ; needs_checkout="$NEEDS_CHECKOUT_D" + checkout_path="$CHECKOUT_PATH_D" ; version_json="$VERSION_D" ;; + esac + if [[ "$is_platform" != "true" || "$needs_checkout" != "true" || -z "$version_json" ]]; then + continue + fi + tag=$(echo "$version_json" | jq -r '.tag') + src_dir="${INPUT_PATH}/${INPUT_SDK}/src/${tag}" + otdfctl_dir="${checkout_path}/otdfctl" + echo "Symlinking freshly checked-out platform otdfctl ${otdfctl_dir} → ${src_dir}" + mkdir -p "$(dirname "$src_dir")" + ln -sfn "$otdfctl_dir" "$src_dir" + done + env: + INPUT_PATH: ${{ inputs.path }} + INPUT_SDK: ${{ inputs.sdk }} + VERSION_A: ${{ steps.resolve.outputs.version-a }} + VERSION_B: ${{ steps.resolve.outputs.version-b }} + VERSION_C: ${{ steps.resolve.outputs.version-c }} + VERSION_D: ${{ steps.resolve.outputs.version-d }} + IS_PLATFORM_A: ${{ steps.check-source.outputs.is-platform-a }} + IS_PLATFORM_B: ${{ steps.check-source.outputs.is-platform-b }} + IS_PLATFORM_C: ${{ steps.check-source.outputs.is-platform-c }} + IS_PLATFORM_D: ${{ steps.check-source.outputs.is-platform-d }} + NEEDS_CHECKOUT_A: ${{ steps.check-source.outputs.needs-checkout-a }} + NEEDS_CHECKOUT_B: ${{ steps.check-source.outputs.needs-checkout-b }} + NEEDS_CHECKOUT_C: ${{ steps.check-source.outputs.needs-checkout-c }} + NEEDS_CHECKOUT_D: ${{ steps.check-source.outputs.needs-checkout-d }} + CHECKOUT_PATH_A: ${{ steps.check-source.outputs.checkout-path-a }} + CHECKOUT_PATH_B: ${{ steps.check-source.outputs.checkout-path-b }} + CHECKOUT_PATH_C: ${{ steps.check-source.outputs.checkout-path-c }} + CHECKOUT_PATH_D: ${{ steps.check-source.outputs.checkout-path-d }} - name: post checkout cleanups if: inputs.sdk == 'java' From 8f783c3c34b612ca51bca1b14003d4560ff51782 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:19:18 -0400 Subject: [PATCH 10/21] fix: address PR review findings for platform otdfctl migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass --source flag through artifact install step so platform-sourced Go versions get the correct module path in .version files - Strip tag infix (e.g., otdfctl/v0.24.0 → v0.24.0) before normalizing in install_go_release to avoid producing invalid version strings - Move step output expansion to env: block in detect-otdfctl to prevent script injection (consistent with 0c1b428) - Narrow except Exception to GitCommandError in registry.py platform tag query so programming bugs propagate instead of being swallowed - Handle StopIteration explicitly in resolve.py with a clear "main branch not found" error message - Validate source parameter in go_module_path() to reject typos - Add symlink target existence checks after ln -sfn in action.yaml - Add ::warning:: annotation when auto-detect fallback skips SHA verification - Fix typo and minor comment/doc improvements Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 4 +++- otdf-sdk-mgr/README.md | 2 +- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 5 +++++ otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 7 ++++--- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 3 ++- otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py | 5 ++++- xtest/setup-cli-tool/action.yaml | 17 +++++++++++++++-- 7 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 58eec1342..fe37450e8 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -89,6 +89,8 @@ jobs: JAVA_REF: "${{ inputs.java-ref }}" # When explicitly set to 'platform', tells the Go resolver to resolve # against opentdf/platform tags instead of the standalone otdfctl repo. + # Auto-detection of the platform source is handled downstream by the + # detect-otdfctl step and setup-cli-tool action. OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input @@ -310,7 +312,6 @@ jobs: - name: Detect platform-embedded otdfctl id: detect-otdfctl run: | - PLATFORM_DIR="${{ steps.run-platform.outputs.platform-working-dir }}" if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" @@ -330,6 +331,7 @@ jobs: fi env: OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} + PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} ######### CHECKOUT JS CLI ############# - name: Configure js-sdk diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md index 818ee4226..4b930eaae 100644 --- a/otdf-sdk-mgr/README.md +++ b/otdf-sdk-mgr/README.md @@ -56,7 +56,7 @@ otdf-sdk-mgr java-fixup ## How Release Installs Work -- **Go**: Writes a `.version` file containing `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`); `cli.sh`/`otdfctl.sh` use `go run @{version}` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. +- **Go**: Writes a `.version` file containing `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`); `cli.sh`/`otdfctl.sh` use `go run @` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases. - **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/` - **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar` diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 1ef2979d2..5d1a4bcf8 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -146,8 +146,13 @@ def go_install_prefix(source: str | None = None) -> str: return GO_INSTALL_PREFIX_STANDALONE +_VALID_GO_SOURCES = {None, "platform"} + + def go_module_path(source: str | None = None) -> str: """Return the Go module path based on source.""" + if source not in _VALID_GO_SOURCES: + raise ValueError(f"Invalid Go source {source!r}; expected one of {_VALID_GO_SOURCES}") if source == "platform": return GO_MODULE_PATH_PLATFORM return GO_MODULE_PATH diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index dc36ad11a..62113e395 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -40,6 +40,9 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) """ go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) + # Strip tag infix (e.g., "otdfctl/v0.24.0" → "v0.24.0") + if "/" in version: + version = version.rsplit("/", 1)[-1] tag = normalize_version(version) module = go_module_path(source) (dist_dir / ".version").write_text(f"{module}@{tag}\n") @@ -142,9 +145,7 @@ def install_java_release(version: str, dist_dir: Path, **_kwargs: object) -> Non } -def install_release( - sdk: str, version: str, dist_name: str | None = None, **kwargs: object -) -> Path: +def install_release(sdk: str, version: str, dist_name: str | None = None, **kwargs: object) -> Path: """Install a released version of an SDK CLI. Args: diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index aae9a4eeb..f95f801f6 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -71,6 +71,7 @@ def fetch_text(url: str) -> str: def list_go_versions() -> list[dict[str, Any]]: """List Go SDK versions from git tags in both standalone and platform repos.""" + import git.exc from git import Git repo = Git() @@ -119,7 +120,7 @@ def list_go_versions() -> list[dict[str, Any]]: "install_method": f"{GO_INSTALL_PREFIX_PLATFORM}@{tag}", "stable": is_stable(version), } - except Exception as e: + except git.exc.GitCommandError as e: print(f"Warning: failed to query platform repo for go tags: {e}", file=sys.stderr) results = list(seen.values()) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py index 8403759eb..e3f264d74 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py @@ -147,7 +147,10 @@ def _annotate(result: ResolveResult) -> ResolveResult: repo = Git() if version == "main" or version == "refs/heads/main": all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")] - sha, _ = next(tag for tag in all_heads if "refs/heads/main" in tag) + try: + sha, _ = next(tag for tag in all_heads if "refs/heads/main" in tag) + except StopIteration: + return {"sdk": sdk, "alias": version, "err": f"main branch not found in {sdk_url}"} return _annotate( { "sdk": sdk, diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 1aa56d986..f9a0c8313 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -2,7 +2,7 @@ name: configure-sdks description: Check out and build one or more SDK and its CLI tool for use by xtest inputs: path: - description: The path to checkout the the SDK source code to; concatenated with branch or tag name. + description: The path to check out the SDK source code to; concatenated with branch or tag name. sdk: description: The SDK to configure; one of go, java, js version-info: @@ -101,9 +101,12 @@ runs: echo "Installing ${{ inputs.sdk }} $tag from registry (release: $release)" # Sanitize tag for use as an env var name (replace non-alphanumeric/underscore with _) tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" + source=$(echo "$row" | jq -r '.source // empty') + source_flag="" + [[ -n "$source" ]] && source_flag="--source $source" if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \ --sdk "${{ inputs.sdk }}" --version "$release" \ - --dist-name "$tag"; then + --dist-name "$tag" $source_flag; then echo " Warning: Artifact installation failed for ${{ inputs.sdk }} $tag" echo " Will fall back to building from source" echo "BUILD_FROM_SOURCE_${tag_sanitized}=true" >> "$GITHUB_ENV" @@ -171,6 +174,8 @@ runs: # Auto-detect fallback: resolver used standalone repo but the # test job detected otdfctl in the platform checkout. Symlink all # head go versions to the existing dir (backward-compatible behavior). + # Note: no SHA comparison here — all head versions are symlinked regardless of commit. + echo "::warning::Auto-detecting platform source for Go version ${tag} (SHA ${sha}). Platform dir SHA: ${PLATFORM_OTDFCTL_SHA:-unknown}. No SHA match verification performed." is_platform=true use_existing=true needs_source=false @@ -209,6 +214,10 @@ runs: echo "Symlinking existing platform otdfctl to ${src_dir}" mkdir -p "$(dirname "$src_dir")" ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir" + if [ ! -e "$src_dir" ]; then + echo "::error::Symlink target does not exist: $PLATFORM_OTDFCTL_DIR" + exit 1 + fi done env: PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }} @@ -293,6 +302,10 @@ runs: echo "Symlinking freshly checked-out platform otdfctl ${otdfctl_dir} → ${src_dir}" mkdir -p "$(dirname "$src_dir")" ln -sfn "$otdfctl_dir" "$src_dir" + if [ ! -e "$src_dir" ]; then + echo "::error::Symlink target does not exist: ${otdfctl_dir} (does the platform repo contain an otdfctl/ directory?)" + exit 1 + fi done env: INPUT_PATH: ${{ inputs.path }} From 1e5fc53ac51f83d9693c53bc226899e04cadbdf3 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:26:08 -0400 Subject: [PATCH 11/21] fix(sdk-mgr): accept "standalone" as valid Go source in go_module_path The workflow input and CLI advertise "standalone" as a valid source, but go_module_path rejected it with a ValueError. Add "standalone" to the valid set so it maps to the default standalone module path, consistent with go_git_url, go_tag_infix, and go_install_prefix. Co-Authored-By: Claude Opus 4.6 (1M context) --- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 5d1a4bcf8..1b71831df 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -146,7 +146,7 @@ def go_install_prefix(source: str | None = None) -> str: return GO_INSTALL_PREFIX_STANDALONE -_VALID_GO_SOURCES = {None, "platform"} +_VALID_GO_SOURCES = {None, "standalone", "platform"} def go_module_path(source: str | None = None) -> str: From 821c02c05142003c5f1695c53dac9131b2903515 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:30:53 -0400 Subject: [PATCH 12/21] docs(xtest): clarify auto mode resolves releases from standalone The otdfctl-source description implied auto detects from the platform checkout, but release resolution runs before that checkout exists. Update the input description and inline comment to accurately reflect the two-phase behavior: releases resolve from standalone, platform detection applies only to head builds. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index fe37450e8..35dc055e1 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -32,7 +32,7 @@ on: required: false type: string default: auto - description: "otdfctl source: 'auto' (detect from platform checkout), 'standalone', or 'platform'" + description: "otdfctl source: 'auto' (standalone for releases, detect platform for head builds), 'standalone', or 'platform'" workflow_call: inputs: platform-ref: @@ -89,8 +89,8 @@ jobs: JAVA_REF: "${{ inputs.java-ref }}" # When explicitly set to 'platform', tells the Go resolver to resolve # against opentdf/platform tags instead of the standalone otdfctl repo. - # Auto-detection of the platform source is handled downstream by the - # detect-otdfctl step and setup-cli-tool action. + # In 'auto' mode, releases resolve from standalone; the detect-otdfctl + # step later checks for platform-embedded otdfctl for head builds only. OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input From 290154484dcc641a74797b4daa8ad2770c8f0550 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:34:13 -0400 Subject: [PATCH 13/21] fix(setup-cli-tool): require SHA match in auto-detect platform fallback In auto mode, setup-cli-tool previously symlinked all head Go versions to the platform checkout without verifying the commit SHA. This meant otdfctl-ref=my-feature could silently test the platform-embedded otdfctl instead of opentdf/otdfctl@my-feature. Now the fallback only reuses the platform checkout when the resolved SHA matches; otherwise it falls through to a standalone checkout. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 4 +++- xtest/setup-cli-tool/action.yaml | 18 +++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 35dc055e1..9f26ba3ac 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -90,7 +90,9 @@ jobs: # When explicitly set to 'platform', tells the Go resolver to resolve # against opentdf/platform tags instead of the standalone otdfctl repo. # In 'auto' mode, releases resolve from standalone; the detect-otdfctl - # step later checks for platform-embedded otdfctl for head builds only. + # step later checks for platform-embedded otdfctl for head builds only, + # and setup-cli-tool reuses the platform checkout only when the resolved + # SHA matches the platform checkout SHA. OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index f9a0c8313..fe4995e4a 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -172,13 +172,17 @@ runs: fi elif [[ "$INPUT_SDK" == "go" && -n "$PLATFORM_OTDFCTL_DIR" && "$needs_source" == "true" ]]; then # Auto-detect fallback: resolver used standalone repo but the - # test job detected otdfctl in the platform checkout. Symlink all - # head go versions to the existing dir (backward-compatible behavior). - # Note: no SHA comparison here — all head versions are symlinked regardless of commit. - echo "::warning::Auto-detecting platform source for Go version ${tag} (SHA ${sha}). Platform dir SHA: ${PLATFORM_OTDFCTL_SHA:-unknown}. No SHA match verification performed." - is_platform=true - use_existing=true - needs_source=false + # test job detected otdfctl in the platform checkout. Only reuse + # the platform dir when the resolved SHA matches; otherwise fall + # through to a standalone checkout so we test the correct code. + if [[ -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then + echo "Auto-detected platform source for Go version ${tag} — SHA ${sha} matches platform checkout." + is_platform=true + use_existing=true + needs_source=false + else + echo "::notice::Go version ${tag} (SHA ${sha}) does not match platform checkout (SHA ${PLATFORM_OTDFCTL_SHA:-unknown}); checking out standalone repo." + fi fi echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT" From 7e99e2d25b3f9f0105e60b3518326a0653844033 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 11:51:20 -0400 Subject: [PATCH 14/21] fix: harden validation and error handling for platform otdfctl migration Validate PLATFORM_DIR before using it in detect-otdfctl to prevent wrong-repo SHA when the directory is missing. Remove dead cross-repo SHA comparison in auto-detect fallback (standalone and platform repos have different commit histories). Add input validation to all go_* config helpers and the OTDFCTL_SOURCE env var. Treat pre-warm failure as a hard error for the platform module path. Use array pattern for source_args to prevent shell word-splitting. Improve observability with ::warning:: annotations and version-override logging. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 11 +++++--- otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py | 9 ++++++- otdf-sdk-mgr/src/otdf_sdk_mgr/config.py | 25 +++++++++-------- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 14 +++++++--- otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py | 14 ++++++++-- xtest/setup-cli-tool/action.yaml | 27 +++++++++---------- 6 files changed, 62 insertions(+), 38 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 9f26ba3ac..766a4dcf1 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -90,9 +90,10 @@ jobs: # When explicitly set to 'platform', tells the Go resolver to resolve # against opentdf/platform tags instead of the standalone otdfctl repo. # In 'auto' mode, releases resolve from standalone; the detect-otdfctl - # step later checks for platform-embedded otdfctl for head builds only, - # and setup-cli-tool reuses the platform checkout only when the resolved - # SHA matches the platform checkout SHA. + # step later probes the platform checkout for an embedded otdfctl + # directory, and setup-cli-tool acts on this only for versions that need + # a source checkout (head or artifact-install failure), reusing the + # platform checkout only when the resolved SHA matches. OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}" steps: - name: Validate focus-sdk input @@ -325,6 +326,10 @@ jobs: echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" fi elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then + if [ -z "$PLATFORM_DIR" ] || [ ! -d "$PLATFORM_DIR/otdfctl" ] || [ ! -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "::error::otdfctl-source=platform requested but ${PLATFORM_DIR:-}/otdfctl does not exist or lacks go.mod" + exit 1 + fi echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py index 1d4d6823a..2dcf6e321 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py @@ -113,8 +113,15 @@ def resolve_versions( raise typer.Exit(2) infix = SDK_TAG_INFIXES.get(sdk) - # Allow overriding the Go SDK source (standalone otdfctl repo vs platform monorepo) + # Allow overriding the Go SDK source via OTDFCTL_SOURCE env var + # (standalone otdfctl repo vs platform monorepo) go_source = os.environ.get("OTDFCTL_SOURCE") if sdk == "go" else None + if go_source and go_source not in ("standalone", "platform"): + typer.echo( + f"Warning: unrecognized OTDFCTL_SOURCE={go_source!r}; expected 'platform' or 'standalone'", + err=True, + ) + go_source = None results: list[ResolveResult] = [] shas: set[str] = set() diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py index 1b71831df..1046a5ef8 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py @@ -116,9 +116,17 @@ def get_sdk_dirs() -> dict[str, Path]: } # When resolving go versions from the platform repo, use "otdfctl" infix -# (tags are otdfctl/v0.X.Y in the platform monorepo) +# (tags are otdfctl/vX.Y.Z in the platform monorepo) SDK_TAG_INFIXES_PLATFORM_GO = "otdfctl" +_VALID_GO_SOURCES = {None, "standalone", "platform"} + + +def _validate_go_source(source: str | None) -> None: + """Raise ValueError if source is not a recognised Go source.""" + if source not in _VALID_GO_SOURCES: + raise ValueError(f"Invalid Go source {source!r}; expected one of {_VALID_GO_SOURCES}") + def go_git_url(source: str | None = None) -> str: """Return the git URL for Go SDK resolution based on source. @@ -127,6 +135,7 @@ def go_git_url(source: str | None = None) -> str: source: "platform" to use the platform monorepo, None/"standalone" for the standalone otdfctl repo. """ + _validate_go_source(source) if source == "platform": return SDK_GIT_URLS["platform"] return SDK_GIT_URLS["go"] @@ -134,25 +143,15 @@ def go_git_url(source: str | None = None) -> str: def go_tag_infix(source: str | None = None) -> str | None: """Return the tag infix for Go SDK resolution based on source.""" + _validate_go_source(source) if source == "platform": return SDK_TAG_INFIXES_PLATFORM_GO return None -def go_install_prefix(source: str | None = None) -> str: - """Return the go install/run prefix based on source.""" - if source == "platform": - return GO_INSTALL_PREFIX_PLATFORM - return GO_INSTALL_PREFIX_STANDALONE - - -_VALID_GO_SOURCES = {None, "standalone", "platform"} - - def go_module_path(source: str | None = None) -> str: """Return the Go module path based on source.""" - if source not in _VALID_GO_SOURCES: - raise ValueError(f"Invalid Go source {source!r}; expected one of {_VALID_GO_SOURCES}") + _validate_go_source(source) if source == "platform": return GO_MODULE_PATH_PLATFORM return GO_MODULE_PATH diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 62113e395..b5e668336 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -11,6 +11,7 @@ from pathlib import Path from otdf_sdk_mgr.config import ( + GO_MODULE_PATH_PLATFORM, LTS_VERSIONS, get_sdk_dir, get_sdk_dirs, @@ -36,7 +37,8 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) Args: version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0"). dist_dir: Target distribution directory. - source: "platform" to use the platform monorepo module path, None for standalone. + source: "platform" to use the platform monorepo module path, + None or "standalone" for standalone. """ go_dir = get_sdk_dir() / "go" dist_dir.mkdir(parents=True, exist_ok=True) @@ -56,9 +58,13 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) text=True, ) if result.returncode != 0: - print( - f" Warning: go install pre-warm failed (will retry at runtime): {result.stderr.strip()}" - ) + msg = f"go install pre-warm failed: {result.stderr.strip()}" + if module == GO_MODULE_PATH_PLATFORM: + raise InstallError( + f"{msg}\n" + f"The platform module path {module}@{tag} may not be published yet." + ) + print(f" Warning: {msg} (will retry at runtime)") print(f" Go release {tag} installed to {dist_dir}") diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py index f95f801f6..fcd4f78c8 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py @@ -112,7 +112,14 @@ def list_go_versions() -> list[dict[str, Any]]: version = tag.removeprefix(f"{infix}/") if not parse_semver(version): continue - # Platform entries take precedence (canonical location post-migration) + # Platform entries take precedence (canonical location post-migration); + # if the same version exists in both repos, the platform entry + # silently overwrites the standalone one. + if version in seen: + print( + f"Note: version {version} found in both standalone and platform repos; using platform source.", + file=sys.stderr, + ) seen[version] = { "sdk": "go", "version": version, @@ -121,7 +128,10 @@ def list_go_versions() -> list[dict[str, Any]]: "stable": is_stable(version), } except git.exc.GitCommandError as e: - print(f"Warning: failed to query platform repo for go tags: {e}", file=sys.stderr) + print( + f"::warning::Failed to query platform repo for go tags: {e}", + file=sys.stderr, + ) results = list(seen.values()) results.sort(key=lambda r: semver_sort_key(r["version"])) diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index fe4995e4a..29747fa4e 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -12,7 +12,8 @@ inputs: description: >- Absolute path to platform's otdfctl/ directory. When set and sdk is "go", head versions whose SHA matches platform-otdfctl-sha are symlinked from - here instead of checked out separately. + here instead of checked out separately. Used in both explicit platform + mode (source: "platform" in resolved version) and auto-detect mode. platform-otdfctl-sha: description: >- SHA of the commit that platform-otdfctl-dir was checked out at. @@ -102,11 +103,11 @@ runs: # Sanitize tag for use as an env var name (replace non-alphanumeric/underscore with _) tag_sanitized="${tag//[^a-zA-Z0-9_]/_}" source=$(echo "$row" | jq -r '.source // empty') - source_flag="" - [[ -n "$source" ]] && source_flag="--source $source" + source_args=() + [[ -n "$source" ]] && source_args=(--source "$source") if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \ --sdk "${{ inputs.sdk }}" --version "$release" \ - --dist-name "$tag" $source_flag; then + --dist-name "$tag" "${source_args[@]}"; then echo " Warning: Artifact installation failed for ${{ inputs.sdk }} $tag" echo " Will fall back to building from source" echo "BUILD_FROM_SOURCE_${tag_sanitized}=true" >> "$GITHUB_ENV" @@ -172,17 +173,13 @@ runs: fi elif [[ "$INPUT_SDK" == "go" && -n "$PLATFORM_OTDFCTL_DIR" && "$needs_source" == "true" ]]; then # Auto-detect fallback: resolver used standalone repo but the - # test job detected otdfctl in the platform checkout. Only reuse - # the platform dir when the resolved SHA matches; otherwise fall - # through to a standalone checkout so we test the correct code. - if [[ -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then - echo "Auto-detected platform source for Go version ${tag} — SHA ${sha} matches platform checkout." - is_platform=true - use_existing=true - needs_source=false - else - echo "::notice::Go version ${tag} (SHA ${sha}) does not match platform checkout (SHA ${PLATFORM_OTDFCTL_SHA:-unknown}); checking out standalone repo." - fi + # test job detected otdfctl in the platform checkout. + # NOTE: SHA comparison across repos is not meaningful (the standalone + # repo and platform repo have different commit histories), so we + # cannot safely reuse the platform checkout here. Fall through to + # a standalone checkout. To use the platform source, set + # otdfctl-source=platform explicitly. + echo "::notice::Go version ${tag} resolved from standalone repo; platform checkout available but cannot auto-reuse (different repo). Set otdfctl-source=platform to use the platform source." fi echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT" From 13385f513457dd23614fe4b7442e0dfdde619cc1 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 12:13:04 -0400 Subject: [PATCH 15/21] fixup ruff format --- otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index b5e668336..0822a063e 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -61,8 +61,7 @@ def install_go_release(version: str, dist_dir: Path, source: str | None = None) msg = f"go install pre-warm failed: {result.stderr.strip()}" if module == GO_MODULE_PATH_PLATFORM: raise InstallError( - f"{msg}\n" - f"The platform module path {module}@{tag} may not be published yet." + f"{msg}\nThe platform module path {module}@{tag} may not be published yet." ) print(f" Warning: {msg} (will retry at runtime)") print(f" Go release {tag} installed to {dist_dir}") From bfe11196652bff3d51dcdebf51fff0adf7cee4f8 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Thu, 16 Apr 2026 21:24:38 -0400 Subject: [PATCH 16/21] refactor(xtest): extract composite actions from xtest.yml Replace ~300 lines of inline workflow steps with three reusable composite actions and two new CLI commands, reducing xtest.yml from 770 to 524 lines. New composite actions: - setup-test-environment: consolidates otdfctl detection, platform version lookup, key management check, root key extraction, and multikas support - setup-sdk-clients: wraps setup-cli-tool with SDK-appropriate caching, go.mod/java .env fixups, and make builds (one call per SDK) - setup-kas-instances: starts all 6 KAS instances via otdf-local ci start-kas New CLI commands: - otdf-sdk-mgr go-fixup: bridges client go.mod to server shared modules for head builds (parallels existing java-fixup) - otdf-local ci start-kas: starts KAS instances in CI and emits GITHUB_OUTPUT lines for log file paths Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 355 ++++------------------ otdf-local/src/otdf_local/ci.py | 223 ++++++++++++++ otdf-local/src/otdf_local/cli.py | 3 + otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py | 36 +++ otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py | 95 ++++++ xtest/setup-kas-instances/action.yaml | 86 ++++++ xtest/setup-sdk-clients/action.yaml | 159 ++++++++++ xtest/setup-test-environment/action.yaml | 140 +++++++++ 8 files changed, 797 insertions(+), 300 deletions(-) create mode 100644 otdf-local/src/otdf_local/ci.py create mode 100644 otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py create mode 100644 xtest/setup-kas-instances/action.yaml create mode 100644 xtest/setup-sdk-clients/action.yaml create mode 100644 xtest/setup-test-environment/action.yaml diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 766a4dcf1..3ad813ba4 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -311,226 +311,56 @@ jobs: with: node-version: "22.x" - ######## DETECT PLATFORM-EMBEDDED OTDFCTL ############# - - name: Detect platform-embedded otdfctl - id: detect-otdfctl - run: | - if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then - if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then - echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" - echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" - echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" - echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" - else - echo "otdfctl not found in platform checkout; using standalone repo" - echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" - fi - elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then - if [ -z "$PLATFORM_DIR" ] || [ ! -d "$PLATFORM_DIR/otdfctl" ] || [ ! -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then - echo "::error::otdfctl-source=platform requested but ${PLATFORM_DIR:-}/otdfctl does not exist or lacks go.mod" - exit 1 - fi - echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" - echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" - echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" - else - echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" - fi - env: - OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source }} - PLATFORM_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} - - ######### CHECKOUT JS CLI ############# - - name: Configure js-sdk - id: configure-js - uses: ./otdftests/xtest/setup-cli-tool + ######## SETUP TEST ENVIRONMENT ############# + - name: Setup test environment + id: test-env + uses: ./otdftests/xtest/setup-test-environment + with: + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + platform-tag: ${{ matrix.platform-tag }} + otdfctl-source-input: ${{ inputs.otdfctl-source }} + + ######## SETUP SDK CLIENTS ############# + - name: Setup JS SDK client + id: setup-js + uses: ./otdftests/xtest/setup-sdk-clients with: - path: otdftests/xtest/sdk sdk: js version-info: "${{ needs.resolve-versions.outputs.js }}" - - - name: Cache npm - if: fromJson(steps.configure-js.outputs.heads)[0] != null - uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + platform-heads: ${{ needs.resolve-versions.outputs.heads }} + platform-tag: ${{ matrix.platform-tag }} + platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }} + + - name: Setup Go SDK client (otdfctl) + id: setup-go + uses: ./otdftests/xtest/setup-sdk-clients with: - path: ~/.npm - key: npm-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/js/src/**/package-lock.json') }} - restore-keys: | - npm-${{ runner.os }}- - - ######## SETUP THE JS CLI ############# - - name: build and setup the web-sdk cli - id: build-web-sdk - if: fromJson(steps.configure-js.outputs.heads)[0] != null - run: | - make - working-directory: otdftests/xtest/sdk/js - - ######## CHECKOUT GO CLI ############# - - name: Configure otdfctl - id: configure-go - uses: ./otdftests/xtest/setup-cli-tool - with: - path: otdftests/xtest/sdk sdk: go version-info: "${{ needs.resolve-versions.outputs.go }}" - platform-otdfctl-dir: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} - platform-otdfctl-sha: ${{ steps.detect-otdfctl.outputs.otdfctl-sha }} - - - name: Cache Go modules - if: fromJson(steps.configure-go.outputs.heads)[0] != null - uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: go-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/go/src/*/go.sum') }} - restore-keys: | - go-${{ runner.os }}- - - - name: Resolve otdfctl heads - id: resolve-otdfctl-heads - if: fromJson(steps.configure-go.outputs.heads)[0] != null - run: |- - echo "OTDFCTL_HEADS=$OTDFCTL_HEADS" >> "$GITHUB_ENV" - env: - OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }} - - - name: Replace otdfctl go.mod packages, but only at head version of platform - if: >- - steps.detect-otdfctl.outputs.otdfctl-source != 'platform' - && fromJson(steps.configure-go.outputs.heads)[0] != null - && env.FOCUS_SDK == 'go' - && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) - env: - PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} - run: |- - echo "Replacing go.mod packages (standalone otdfctl)..." - PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}" - OTDFCTL_DIR_ABS="$(pwd)/otdftests/xtest/sdk/go/src/" - echo "PLATFORM_DIR_ABS: $PLATFORM_DIR_ABS" - echo "OTDFCTL_DIR_ABS: $OTDFCTL_DIR_ABS" - for head in $(echo "${OTDFCTL_HEADS}" | jq -r '.[]'); do - echo "Processing head: $head" - cd "${OTDFCTL_DIR_ABS}/$head" - for m in lib/fixtures lib/ocrypto protocol/go sdk; do - go mod edit -replace "github.com/opentdf/platform/$m=${PLATFORM_DIR_ABS}/$m" - done - go mod tidy - done - - ######## SETUP THE GO CLI ############# - - name: Prepare go cli - if: fromJson(steps.configure-go.outputs.heads)[0] != null - run: |- - make - working-directory: otdftests/xtest/sdk/go - - ####### CHECKOUT JAVA SDK ############## - - - name: Configure java-sdk - id: configure-java - uses: ./otdftests/xtest/setup-cli-tool + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + platform-heads: ${{ needs.resolve-versions.outputs.heads }} + platform-tag: ${{ matrix.platform-tag }} + platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }} + otdfctl-source: ${{ steps.test-env.outputs.otdfctl-source }} + otdfctl-dir: ${{ steps.test-env.outputs.otdfctl-dir }} + otdfctl-sha: ${{ steps.test-env.outputs.otdfctl-sha }} + focus-sdk: ${{ inputs.focus-sdk || 'all' }} + + - name: Setup Java SDK client + id: setup-java + uses: ./otdftests/xtest/setup-sdk-clients with: - path: otdftests/xtest/sdk sdk: java version-info: "${{ needs.resolve-versions.outputs.java }}" - - - name: Cache Maven repository - if: fromJson(steps.configure-java.outputs.heads)[0] != null - uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 - with: - path: ~/.m2/repository - key: maven-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/java/src/**/pom.xml') }} - restore-keys: | - maven-${{ runner.os }}- - - - name: pre-release protocol buffers for java-sdk - if: >- - fromJson(steps.configure-java.outputs.heads)[0] != null - && (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java') - && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) - run: |- - echo "Replacing .env files for java-sdk..." - echo "Platform tag: $platform_tag" - echo "Java version info: $java_version_info" - for row in $(echo "$java_version_info" | jq -c '.[]'); do - TAG=$(echo "$row" | jq -r '.tag') - HEAD=$(echo "$row" | jq -r '.head') - if [[ "$HEAD" == "true" ]]; then - echo "Creating .env file for tag: [$TAG]; pointing to platform ref [$platform_tag]" - echo "PLATFORM_BRANCH=$platform_ref" > "otdftests/xtest/sdk/java/${TAG}.env" - else - echo "Skipping .env file creation for release version [$TAG]" - fi - done - env: - java_version_info: ${{ needs.resolve-versions.outputs.java }} - platform_ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)[matrix.platform-tag] }} - platform_tag: ${{ matrix.platform-tag }} - - ####### SETUP JAVA CLI ############## - - name: Prepare java cli - if: fromJson(steps.configure-java.outputs.heads)[0] != null - run: | - make - working-directory: otdftests/xtest/sdk/java - env: - BUF_INPUT_HTTPS_USERNAME: opentdf-bot - BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} - - ######## Configure test environment ############# - - name: Lookup current platform version - id: platform-version - run: |- - if ! go run ./service version; then - # NOTE: the version command was added in 0.4.37 - echo "Error: Unable to get platform version; defaulting to tag: [$PLATFORM_TAG]" - echo "PLATFORM_VERSION=$PLATFORM_TAG" >> "$GITHUB_ENV" - exit - fi - # Older version commands output version to stderr; newer versions output to stdout - PLATFORM_VERSION=$(go run ./service version 2>&1) - echo "PLATFORM_VERSION=$PLATFORM_VERSION" >> "$GITHUB_ENV" - echo "## Platform version output: [$PLATFORM_VERSION]" - working-directory: ${{ steps.run-platform.outputs.platform-working-dir }} - env: - PLATFORM_TAG: ${{ matrix.platform-tag }} - - - name: Check key management support and prepare root key - id: km-check - run: |- - OT_CONFIG_FILE="$(pwd)/opentdf.yaml" - echo "OT_CONFIG_FILE=$OT_CONFIG_FILE" >> "$GITHUB_ENV" - # Determine if the config declares the key_management field - km_value=$(yq e '.services.kas.preview.key_management' "$OT_CONFIG_FILE" 2>/dev/null || echo "null") - case "$km_value" in - true|false) - echo "KEY_MANAGEMENT_SUPPORTED=true" >> "$GITHUB_ENV" - echo "supported=true" >> "$GITHUB_OUTPUT" - ;; - *) - echo "KEY_MANAGEMENT_SUPPORTED=false" >> "$GITHUB_ENV" - echo "supported=false" >> "$GITHUB_OUTPUT" - ;; - esac - # Prepare a root key for use by additional KAS instances - existing_root_key=$(yq e '.services.kas.root_key' "$OT_CONFIG_FILE" 2>/dev/null || echo "") - if [ -n "$existing_root_key" ] && [ "$existing_root_key" != "null" ]; then - echo "Using existing root key from config" - echo "OT_ROOT_KEY=$existing_root_key" >> "$GITHUB_ENV" - echo "root_key=$existing_root_key" >> "$GITHUB_OUTPUT" - else - echo "Generating a new root key for additional KAS" - gen_root_key=$(openssl rand -hex 32) - echo "OT_ROOT_KEY=$gen_root_key" >> "$GITHUB_ENV" - echo "root_key=$gen_root_key" >> "$GITHUB_OUTPUT" - fi - working-directory: ${{ steps.run-platform.outputs.platform-working-dir }} - - - name: Install test dependencies - run: uv sync - working-directory: otdftests/xtest + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + platform-heads: ${{ needs.resolve-versions.outputs.heads }} + platform-tag: ${{ matrix.platform-tag }} + platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }} + focus-sdk: ${{ inputs.focus-sdk || 'all' }} + pat-opentdf: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }} + + ######## VALIDATE HELPERS ############# - name: Validate xtest helper library (tests of the test harness and its utilities) if: ${{ !inputs }} run: |- @@ -579,92 +409,17 @@ jobs: ######## ATTRIBUTE BASED CONFIGURATION ############# - - name: Does platform support multikas? - id: multikas - run: |- - if [[ $PLATFORM_TAG == main ]]; then - echo "Main supports multikas" - echo "supported=true" >> "$GITHUB_OUTPUT" - elif awk -F. '{ if ($1 > 0 || ($1 == 0 && $2 > 4)) exit 0; else exit 1; }' <<< "${PLATFORM_VERSION#v}"; then - echo "Selected version [$PLATFORM_VERSION] supports multikas" - echo "supported=true" >> "$GITHUB_OUTPUT" - else - echo "At tag [$PLATFORM_TAG], [$PLATFORM_VERSION] probably does not support multikas" - echo "supported=false" >> "$GITHUB_OUTPUT" - fi - env: - PLATFORM_TAG: ${{ matrix.platform-tag }} - - - name: Start additional kas - id: kas-alpha - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - kas-name: alpha - kas-port: 8181 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional kas - id: kas-beta - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - kas-name: beta - kas-port: 8282 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional kas - id: kas-gamma - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - kas-name: gamma - kas-port: 8383 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional kas - id: kas-delta - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix + - name: Start KAS instances for ABAC tests + id: kas-instances + if: ${{ steps.test-env.outputs.multikas-supported == 'true' }} + uses: ./otdftests/xtest/setup-kas-instances with: - ec-tdf-enabled: true - kas-port: 8484 - kas-name: delta - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional KM kas (km1) - id: kas-km1 - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - key-management: ${{ steps.km-check.outputs.supported }} - kas-name: km1 - kas-port: 8585 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} - - - name: Start additional KM kas (km2) - id: kas-km2 - if: ${{ steps.multikas.outputs.supported == 'true' }} - uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix - with: - ec-tdf-enabled: true - kas-name: km2 - key-management: ${{ steps.km-check.outputs.supported }} - kas-port: 8686 - log-type: json - root-key: ${{ steps.km-check.outputs.root_key }} + platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }} + root-key: ${{ steps.test-env.outputs.root-key }} + key-management-supported: ${{ steps.test-env.outputs.key-management-supported }} - name: Run attribute based configuration tests - if: ${{ steps.multikas.outputs.supported == 'true' }} + if: ${{ steps.test-env.outputs.multikas-supported == 'true' }} run: >- uv run pytest -ra @@ -682,12 +437,12 @@ jobs: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" PLATFORM_TAG: ${{ matrix.platform-tag }} PLATFORM_LOG_FILE: "../../${{ steps.run-platform.outputs.platform-log-file }}" - KAS_ALPHA_LOG_FILE: "../../${{ steps.kas-alpha.outputs.log-file }}" - KAS_BETA_LOG_FILE: "../../${{ steps.kas-beta.outputs.log-file }}" - KAS_GAMMA_LOG_FILE: "../../${{ steps.kas-gamma.outputs.log-file }}" - KAS_DELTA_LOG_FILE: "../../${{ steps.kas-delta.outputs.log-file }}" - KAS_KM1_LOG_FILE: "../../${{ steps.kas-km1.outputs.log-file }}" - KAS_KM2_LOG_FILE: "../../${{ steps.kas-km2.outputs.log-file }}" + KAS_ALPHA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-alpha-log-file }} + KAS_BETA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-beta-log-file }} + KAS_GAMMA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-gamma-log-file }} + KAS_DELTA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-delta-log-file }} + KAS_KM1_LOG_FILE: ${{ steps.kas-instances.outputs.kas-km1-log-file }} + KAS_KM2_LOG_FILE: ${{ steps.kas-instances.outputs.kas-km2-log-file }} - name: Upload artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 diff --git a/otdf-local/src/otdf_local/ci.py b/otdf-local/src/otdf_local/ci.py new file mode 100644 index 000000000..bab3b3203 --- /dev/null +++ b/otdf-local/src/otdf_local/ci.py @@ -0,0 +1,223 @@ +"""CI-specific commands for otdf-local. + +These commands adapt the local environment management for GitHub Actions CI, +where the platform is already started by an external action and we only need +to start KAS instances as background processes. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Annotated + +import typer + +from otdf_local.config.ports import Ports +from otdf_local.config.settings import Settings +from otdf_local.health.waits import WaitTimeoutError, wait_for_health +from otdf_local.services import get_kas_manager +from otdf_local.utils.console import ( + print_error, + print_info, + print_success, + print_warning, +) +from otdf_local.utils.yaml import load_yaml, save_yaml, set_nested + +ci_app = typer.Typer( + name="ci", + help="CI-specific commands for GitHub Actions workflows.", + no_args_is_help=True, +) + + +def _emit_github_output(key: str, value: str) -> None: + """Write a key=value pair to $GITHUB_OUTPUT if available, else print to stdout.""" + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + with open(github_output, "a") as f: + f.write(f"{key}={value}\n") + else: + # Fallback for local testing + print(f"{key}={value}", file=sys.stdout) + + +def _prepare_kas_template( + settings: Settings, root_key: str | None, ec_tdf_enabled: bool +) -> None: + """Ensure the KAS template config has the right root key and EC TDF settings. + + In CI, the platform config may have a root_key that differs from what + we want for additional KAS instances. This updates the platform config + in-place so that KASService._generate_config reads the correct root_key. + """ + if root_key: + config = load_yaml(settings.platform_config) + set_nested(config, "services.kas.root_key", root_key) + if ec_tdf_enabled: + set_nested(config, "services.kas.preview.ec_tdf_enabled", True) + save_yaml(settings.platform_config, config) + + +@ci_app.command("start-kas") +def start_kas( + platform_dir: Annotated[ + Path, + typer.Option( + "--platform-dir", + help="Path to the platform checkout (must contain opentdf-kas-mode.yaml)", + envvar="OTDF_LOCAL_PLATFORM_DIR", + ), + ], + root_key: Annotated[ + str | None, + typer.Option( + "--root-key", + help="Root key for KAS instances (overrides platform config value)", + envvar="OT_ROOT_KEY", + ), + ] = None, + ec_tdf_enabled: Annotated[ + bool, + typer.Option( + "--ec-tdf-enabled/--no-ec-tdf", + help="Enable EC TDF support", + ), + ] = True, + key_management: Annotated[ + bool, + typer.Option( + "--key-management/--no-key-management", + help="Enable key management on km1/km2 instances", + ), + ] = False, + log_type: Annotated[ + str, + typer.Option( + "--log-type", + help="Log format type (json, text)", + ), + ] = "json", + health_timeout: Annotated[ + int, + typer.Option( + "--health-timeout", + help="Seconds to wait for each KAS instance to become healthy", + ), + ] = 60, + instances: Annotated[ + str | None, + typer.Option( + "--instances", + help="Comma-separated KAS instance names (default: all)", + ), + ] = None, +) -> None: + """Start KAS instances for CI and emit GitHub Actions outputs. + + Expects the platform to already be running (started by start-up-with-containers). + Starts all 6 KAS instances (alpha, beta, gamma, delta, km1, km2) as background + processes, waits for each to pass health checks, and emits log file paths as + GitHub Actions step outputs. + + Output keys (written to $GITHUB_OUTPUT): + kas-alpha-log-file, kas-beta-log-file, kas-gamma-log-file, + kas-delta-log-file, kas-km1-log-file, kas-km2-log-file + """ + platform_dir = platform_dir.resolve() + if not platform_dir.is_dir(): + print_error(f"Platform directory does not exist: {platform_dir}") + raise typer.Exit(1) + + # Check for required template files + kas_template = platform_dir / "opentdf-kas-mode.yaml" + platform_config = platform_dir / "opentdf-dev.yaml" + if not kas_template.exists(): + # Fall back to opentdf.yaml if opentdf-kas-mode.yaml doesn't exist + kas_template_alt = platform_dir / "opentdf.yaml" + if kas_template_alt.exists(): + print_info( + f"Using {kas_template_alt} as KAS template (opentdf-kas-mode.yaml not found)" + ) + else: + print_error( + f"Neither opentdf-kas-mode.yaml nor opentdf.yaml found in {platform_dir}" + ) + raise typer.Exit(1) + + if not platform_config.exists(): + # Try opentdf.yaml as fallback + platform_config_alt = platform_dir / "opentdf.yaml" + if platform_config_alt.exists(): + platform_config = platform_config_alt + + # Build settings with CI-specific overrides + # We use a fresh xtest_root derived from this package's location + settings = Settings( + platform_dir=platform_dir, + ) + settings.ensure_directories() + + # Update root key in platform config if provided + if root_key: + _prepare_kas_template(settings, root_key, ec_tdf_enabled) + + # Determine which instances to start + if instances: + kas_names = [n.strip() for n in instances.split(",")] + for name in kas_names: + if name not in Ports.all_kas_names(): + print_error(f"Unknown KAS instance: {name}") + raise typer.Exit(1) + else: + kas_names = Ports.all_kas_names() + + # Start KAS instances + print_info(f"Starting KAS instances: {', '.join(kas_names)}...") + kas_manager = get_kas_manager(settings) + + failed = [] + for name in kas_names: + kas = kas_manager.get(name) + if kas is None: + print_error(f"KAS instance {name} not found in manager") + failed.append(name) + continue + if not kas.start(): + print_error(f"Failed to start KAS {name}") + failed.append(name) + + if failed: + print_error(f"Failed to start: {', '.join(failed)}") + raise typer.Exit(1) + + # Wait for health + print_info("Waiting for KAS health checks...") + unhealthy = [] + for name in kas_names: + port = Ports.get_kas_port(name) + try: + wait_for_health( + f"http://localhost:{port}/healthz", + timeout=health_timeout, + service_name=f"KAS {name}", + ) + except WaitTimeoutError as e: + print_warning(str(e)) + unhealthy.append(name) + + if unhealthy: + print_error(f"KAS instances failed health check: {', '.join(unhealthy)}") + raise typer.Exit(1) + + print_success(f"All {len(kas_names)} KAS instances are healthy") + + # Emit outputs + for name in kas_names: + log_path = settings.get_kas_log_path(name) + output_key = f"kas-{name}-log-file" + _emit_github_output(output_key, str(log_path)) + + print_success("CI KAS startup complete") diff --git a/otdf-local/src/otdf_local/cli.py b/otdf-local/src/otdf_local/cli.py index d8e3597ff..59fc8e098 100644 --- a/otdf-local/src/otdf_local/cli.py +++ b/otdf-local/src/otdf_local/cli.py @@ -11,6 +11,7 @@ from rich.live import Live from otdf_local import __version__ +from otdf_local.ci import ci_app from otdf_local.config.ports import Ports from otdf_local.config.settings import get_settings from otdf_local.health.waits import WaitTimeoutError, wait_for_health, wait_for_port @@ -43,6 +44,8 @@ pretty_exceptions_enable=sys.stderr.isatty(), ) +app.add_typer(ci_app, name="ci") + def _show_provision_error(result: ProvisionResult, target: str) -> None: """Display provisioning error with stderr details.""" diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py index 24148bdd7..62580ebc7 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py @@ -93,3 +93,39 @@ def java_fixup( from otdf_sdk_mgr.java_fixup import post_checkout_java_fixup post_checkout_java_fixup(base_dir) + + +@app.command("go-fixup") +def go_fixup_cmd( + platform_dir: Annotated[ + Path, + typer.Option("--platform-dir", help="Path to the platform checkout root"), + ], + heads: Annotated[ + Optional[str], + typer.Option( + "--heads", + help="JSON list of head version tags to process (e.g. '[\"main\"]')", + ), + ] = None, + base_dir: Annotated[ + Optional[Path], + typer.Argument(help="Base directory for Go source trees"), + ] = None, +) -> None: + """Bridge Go client go.mod to server shared modules for head builds. + + Performs go mod edit -replace + go mod tidy for each head version, + pointing platform module imports at the local platform checkout. + Only needed for standalone otdfctl checkouts. + """ + import json as json_mod + + from otdf_sdk_mgr.go_fixup import go_fixup + + heads_list = json_mod.loads(heads) if heads else None + try: + go_fixup(platform_dir, heads=heads_list, base_dir=base_dir) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) from e diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py new file mode 100644 index 000000000..4098361c2 --- /dev/null +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py @@ -0,0 +1,95 @@ +"""Post-checkout fixups for Go SDK (otdfctl) source trees. + +Bridges client go.mod to server shared modules for head builds where +client and server share unreleased code. Only applies to standalone +otdfctl checkouts — platform-source builds already have the modules. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from otdf_sdk_mgr.config import get_sdk_dir + +# Platform modules that standalone otdfctl imports and that may need +# a local replace directive when testing against a head platform build. +PLATFORM_MODULES = [ + "lib/fixtures", + "lib/ocrypto", + "protocol/go", + "sdk", +] + + +def go_fixup( + platform_dir: Path, + heads: list[str] | None = None, + base_dir: Path | None = None, +) -> None: + """Replace go.mod references to point at local platform checkout. + + Args: + platform_dir: Absolute path to the platform checkout root + (containing lib/, protocol/, sdk/). + heads: JSON-decoded list of head version tags to process. + If None, all subdirectories under *base_dir* are processed. + base_dir: Directory containing per-version otdfctl source trees + (e.g. ``xtest/sdk/go/src``). Defaults to ``get_sdk_dir() / "go" / "src"``. + """ + if base_dir is None: + base_dir = get_sdk_dir() / "go" / "src" + + if not base_dir.exists(): + print(f"Base directory {base_dir} does not exist, nothing to fix.") + return + + platform_dir = platform_dir.resolve() + if not platform_dir.is_dir(): + raise FileNotFoundError(f"Platform directory does not exist: {platform_dir}") + + dirs_to_process: list[Path] = [] + if heads: + for tag in heads: + d = base_dir / tag + if d.is_dir(): + dirs_to_process.append(d) + else: + print(f"Warning: head directory {d} does not exist, skipping.") + else: + for d in sorted(base_dir.iterdir()): + if d.is_dir() and not d.name.endswith(".git"): + dirs_to_process.append(d) + + if not dirs_to_process: + print("No directories to process.") + return + + for src_dir in dirs_to_process: + if not (src_dir / "go.mod").exists(): + print(f"No go.mod in {src_dir}, skipping.") + continue + + print(f"Applying go.mod replacements in {src_dir}...") + for module in PLATFORM_MODULES: + local_path = platform_dir / module + if not local_path.is_dir(): + print(f" Warning: {local_path} does not exist, skipping {module}") + continue + subprocess.run( + [ + "go", + "mod", + "edit", + "-replace", + f"github.com/opentdf/platform/{module}={local_path}", + ], + cwd=src_dir, + check=True, + ) + print(f" Replaced github.com/opentdf/platform/{module} -> {local_path}") + + print(f"Running go mod tidy in {src_dir}...") + subprocess.run(["go", "mod", "tidy"], cwd=src_dir, check=True) + + print("Go fixup complete.") diff --git a/xtest/setup-kas-instances/action.yaml b/xtest/setup-kas-instances/action.yaml new file mode 100644 index 000000000..f5e420172 --- /dev/null +++ b/xtest/setup-kas-instances/action.yaml @@ -0,0 +1,86 @@ +name: setup-kas-instances +description: >- + Start additional KAS instances for multi-KAS / ABAC tests. + Uses otdf-local ci start-kas to start all 6 KAS instances + (alpha, beta, gamma, delta, km1, km2) and expose their log file paths. + +inputs: + platform-working-dir: + description: Path to the platform checkout directory + required: true + root-key: + description: Root key for KAS instances + required: true + key-management-supported: + description: Enable key management on km1/km2 instances (true/false) + required: false + default: "false" + ec-tdf-enabled: + description: Enable EC TDF support + required: false + default: "true" + log-type: + description: Log format type + required: false + default: "json" + tests-path: + description: Path to the tests repo checkout + required: false + default: "otdftests" + +outputs: + kas-alpha-log-file: + description: Path to KAS alpha log file + value: ${{ steps.start-kas.outputs.kas-alpha-log-file }} + kas-beta-log-file: + description: Path to KAS beta log file + value: ${{ steps.start-kas.outputs.kas-beta-log-file }} + kas-gamma-log-file: + description: Path to KAS gamma log file + value: ${{ steps.start-kas.outputs.kas-gamma-log-file }} + kas-delta-log-file: + description: Path to KAS delta log file + value: ${{ steps.start-kas.outputs.kas-delta-log-file }} + kas-km1-log-file: + description: Path to KAS km1 log file + value: ${{ steps.start-kas.outputs.kas-km1-log-file }} + kas-km2-log-file: + description: Path to KAS km2 log file + value: ${{ steps.start-kas.outputs.kas-km2-log-file }} + +runs: + using: composite + steps: + - name: Start KAS instances + id: start-kas + shell: bash + run: | + KM_FLAG="" + if [[ "$KEY_MANAGEMENT" == "true" ]]; then + KM_FLAG="--key-management" + else + KM_FLAG="--no-key-management" + fi + + EC_FLAG="" + if [[ "$EC_TDF_ENABLED" == "true" ]]; then + EC_FLAG="--ec-tdf-enabled" + else + EC_FLAG="--no-ec-tdf" + fi + + OTDF_LOCAL_DIR="$(cd "$TESTS_PATH" && pwd)/otdf-local" + + uv run --project "$OTDF_LOCAL_DIR" otdf-local ci start-kas \ + --platform-dir "$(pwd)/$PLATFORM_DIR" \ + --root-key "$ROOT_KEY" \ + $EC_FLAG \ + $KM_FLAG \ + --log-type "$LOG_TYPE" + env: + PLATFORM_DIR: ${{ inputs.platform-working-dir }} + ROOT_KEY: ${{ inputs.root-key }} + KEY_MANAGEMENT: ${{ inputs.key-management-supported }} + EC_TDF_ENABLED: ${{ inputs.ec-tdf-enabled }} + LOG_TYPE: ${{ inputs.log-type }} + TESTS_PATH: ${{ inputs.tests-path }} diff --git a/xtest/setup-sdk-clients/action.yaml b/xtest/setup-sdk-clients/action.yaml new file mode 100644 index 000000000..35be35b41 --- /dev/null +++ b/xtest/setup-sdk-clients/action.yaml @@ -0,0 +1,159 @@ +name: setup-sdk-clients +description: >- + Configure, cache, patch, and build an SDK CLI for xtest. Wraps setup-cli-tool + and adds SDK-appropriate caching, go.mod/java .env fixups, and make builds. + Each invocation handles one SDK (go, java, or js). + +inputs: + sdk: + description: "SDK to set up: go, java, or js" + required: true + version-info: + description: JSON-encoded output of otdf-sdk-mgr versions resolve for this SDK + required: true + tests-path: + description: Path to the tests repo checkout + required: false + default: "otdftests" + platform-working-dir: + description: >- + Platform checkout directory. Used for go-fixup (bridging client go.mod + to server shared modules) and detecting platform-embedded otdfctl. + required: false + platform-heads: + description: JSON list of platform tags that are heads (from resolve-versions) + required: false + default: "[]" + platform-tag: + description: Current matrix platform-tag value + required: false + platform-tag-to-sha: + description: JSON object mapping platform tags to SHAs + required: false + default: "{}" + otdfctl-source: + description: "Resolved otdfctl source: platform or standalone" + required: false + default: "standalone" + otdfctl-dir: + description: Absolute path to platform's otdfctl directory + required: false + otdfctl-sha: + description: SHA of the platform otdfctl checkout + required: false + focus-sdk: + description: "SDK focus filter: all, go, java, or js" + required: false + default: "all" + buf-token: + description: BUF token for Java proto compilation + required: false + pat-opentdf: + description: PAT for buf HTTPS password (Java SDK build) + required: false + +outputs: + heads: + description: JSON list of head tags for this SDK + value: ${{ steps.configure.outputs.heads }} + +runs: + using: composite + steps: + # Step 1: Configure SDK via setup-cli-tool (checkout/install) + - name: Configure ${{ inputs.sdk }} + id: configure + uses: ./otdftests/xtest/setup-cli-tool + with: + path: ${{ inputs.tests-path }}/xtest/sdk + sdk: ${{ inputs.sdk }} + version-info: ${{ inputs.version-info }} + platform-otdfctl-dir: ${{ inputs.otdfctl-dir }} + platform-otdfctl-sha: ${{ inputs.otdfctl-sha }} + + # Step 2: SDK-appropriate dependency caching + - name: Cache npm + if: inputs.sdk == 'js' && fromJson(steps.configure.outputs.heads)[0] != null + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: ~/.npm + key: npm-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/js/src/**/package-lock.json', inputs.tests-path)) }} + restore-keys: | + npm-${{ runner.os }}- + + - name: Cache Go modules + if: inputs.sdk == 'go' && fromJson(steps.configure.outputs.heads)[0] != null + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/go/src/*/go.sum', inputs.tests-path)) }} + restore-keys: | + go-${{ runner.os }}- + + - name: Cache Maven repository + if: inputs.sdk == 'java' && fromJson(steps.configure.outputs.heads)[0] != null + uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2 + with: + path: ~/.m2/repository + key: maven-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/java/src/**/pom.xml', inputs.tests-path)) }} + restore-keys: | + maven-${{ runner.os }}- + + # Step 3: SDK-specific fixups + + # Go: Bridge client go.mod to server shared modules (standalone otdfctl only) + - name: Go fixup - replace go.mod packages + if: >- + inputs.sdk == 'go' + && steps.configure.outputs.heads != '[]' + && inputs.otdfctl-source != 'platform' + && inputs.focus-sdk == 'go' + && contains(fromJSON(inputs.platform-heads), inputs.platform-tag) + && inputs.platform-working-dir != '' + shell: bash + run: | + SDK_MGR_DIR="$(cd "$TESTS_PATH" && pwd)/otdf-sdk-mgr" + PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}" + BASE_DIR="$(pwd)/${TESTS_PATH}/xtest/sdk/go/src" + HEADS='${{ steps.configure.outputs.heads }}' + uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr go-fixup \ + --platform-dir "$PLATFORM_DIR_ABS" \ + --heads "$HEADS" \ + "$BASE_DIR" + env: + PLATFORM_WORKING_DIR: ${{ inputs.platform-working-dir }} + TESTS_PATH: ${{ inputs.tests-path }} + + # Java: Create .env files with PLATFORM_BRANCH for head versions + - name: Java fixup - create platform branch .env files + if: >- + inputs.sdk == 'java' + && steps.configure.outputs.heads != '[]' + && (inputs.focus-sdk == 'go' || inputs.focus-sdk == 'java') + && contains(fromJSON(inputs.platform-heads), inputs.platform-tag) + shell: bash + run: | + for row in $(echo "$VERSION_INFO" | jq -c '.[]'); do + TAG=$(echo "$row" | jq -r '.tag') + HEAD=$(echo "$row" | jq -r '.head') + if [[ "$HEAD" == "true" ]]; then + echo "Creating .env file for tag: [$TAG]; pointing to platform ref [$PLATFORM_REF]" + echo "PLATFORM_BRANCH=$PLATFORM_REF" > "${TESTS_PATH}/xtest/sdk/java/${TAG}.env" + fi + done + env: + VERSION_INFO: ${{ inputs.version-info }} + PLATFORM_REF: ${{ fromJSON(inputs.platform-tag-to-sha)[inputs.platform-tag] }} + TESTS_PATH: ${{ inputs.tests-path }} + + # Step 4: Build the SDK CLI + - name: Build ${{ inputs.sdk }} CLI + if: fromJson(steps.configure.outputs.heads)[0] != null + shell: bash + run: make + working-directory: ${{ inputs.tests-path }}/xtest/sdk/${{ inputs.sdk }} + env: + BUF_INPUT_HTTPS_USERNAME: ${{ inputs.sdk == 'java' && 'opentdf-bot' || '' }} + BUF_INPUT_HTTPS_PASSWORD: ${{ inputs.sdk == 'java' && inputs.pat-opentdf || '' }} diff --git a/xtest/setup-test-environment/action.yaml b/xtest/setup-test-environment/action.yaml new file mode 100644 index 000000000..0f8ce86f3 --- /dev/null +++ b/xtest/setup-test-environment/action.yaml @@ -0,0 +1,140 @@ +name: setup-test-environment +description: >- + Detect platform capabilities, extract configuration, and prepare the test + environment. Consolidates otdfctl detection, platform version lookup, key + management support, root key extraction, multikas support check, and test + dependency installation. + +inputs: + platform-working-dir: + description: Platform checkout directory (from start-up-with-containers output) + required: true + platform-tag: + description: Platform version tag under test (matrix value) + required: true + otdfctl-source-input: + description: "User's otdfctl-source preference: auto, standalone, or platform" + required: false + default: "auto" + tests-path: + description: Path to the tests repo checkout + required: false + default: "otdftests" + +outputs: + otdfctl-source: + description: "Resolved otdfctl source: platform or standalone" + value: ${{ steps.detect-otdfctl.outputs.otdfctl-source }} + otdfctl-dir: + description: Absolute path to otdfctl directory (if source=platform) + value: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }} + otdfctl-sha: + description: SHA of the platform checkout (if source=platform) + value: ${{ steps.detect-otdfctl.outputs.otdfctl-sha }} + platform-version: + description: Detected platform version string + value: ${{ steps.platform-version.outputs.version }} + key-management-supported: + description: Whether the platform supports key management (true/false) + value: ${{ steps.km-check.outputs.supported }} + root-key: + description: Root key for KAS instances + value: ${{ steps.km-check.outputs.root_key }} + multikas-supported: + description: Whether multi-KAS is supported (true/false) + value: ${{ steps.multikas.outputs.supported }} + +runs: + using: composite + steps: + - name: Detect platform-embedded otdfctl + id: detect-otdfctl + shell: bash + run: | + if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then + if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl" + echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" + echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" + else + echo "otdfctl not found in platform checkout; using standalone repo" + echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + fi + elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then + if [ -z "$PLATFORM_DIR" ] || [ ! -d "$PLATFORM_DIR/otdfctl" ] || [ ! -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then + echo "::error::otdfctl-source=platform requested but ${PLATFORM_DIR:-}/otdfctl does not exist or lacks go.mod" + exit 1 + fi + echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT" + echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT" + echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT" + else + echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT" + fi + env: + OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source-input }} + PLATFORM_DIR: ${{ inputs.platform-working-dir }} + + - name: Lookup platform version + id: platform-version + shell: bash + run: | + if ! go run ./service version; then + echo "Error: Unable to get platform version; defaulting to tag: [$PLATFORM_TAG]" + echo "version=$PLATFORM_TAG" >> "$GITHUB_OUTPUT" + exit + fi + PLATFORM_VERSION=$(go run ./service version 2>&1) + echo "version=$PLATFORM_VERSION" >> "$GITHUB_OUTPUT" + working-directory: ${{ inputs.platform-working-dir }} + env: + PLATFORM_TAG: ${{ inputs.platform-tag }} + + - name: Check key management support and prepare root key + id: km-check + shell: bash + run: | + OT_CONFIG_FILE="$(pwd)/opentdf.yaml" + km_value=$(yq e '.services.kas.preview.key_management' "$OT_CONFIG_FILE" 2>/dev/null || echo "null") + case "$km_value" in + true|false) + echo "supported=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "supported=false" >> "$GITHUB_OUTPUT" + ;; + esac + existing_root_key=$(yq e '.services.kas.root_key' "$OT_CONFIG_FILE" 2>/dev/null || echo "") + if [ -n "$existing_root_key" ] && [ "$existing_root_key" != "null" ]; then + echo "Using existing root key from config" + echo "root_key=$existing_root_key" >> "$GITHUB_OUTPUT" + else + echo "Generating a new root key for additional KAS" + gen_root_key=$(openssl rand -hex 32) + echo "root_key=$gen_root_key" >> "$GITHUB_OUTPUT" + fi + working-directory: ${{ inputs.platform-working-dir }} + + - name: Check multikas support + id: multikas + shell: bash + run: | + if [[ $PLATFORM_TAG == main ]]; then + echo "Main supports multikas" + echo "supported=true" >> "$GITHUB_OUTPUT" + elif awk -F. '{ if ($1 > 0 || ($1 == 0 && $2 > 4)) exit 0; else exit 1; }' <<< "${PLATFORM_VERSION#v}"; then + echo "Selected version [$PLATFORM_VERSION] supports multikas" + echo "supported=true" >> "$GITHUB_OUTPUT" + else + echo "At tag [$PLATFORM_TAG], [$PLATFORM_VERSION] probably does not support multikas" + echo "supported=false" >> "$GITHUB_OUTPUT" + fi + env: + PLATFORM_TAG: ${{ inputs.platform-tag }} + PLATFORM_VERSION: ${{ steps.platform-version.outputs.version }} + + - name: Install test dependencies + shell: bash + run: uv sync + working-directory: ${{ inputs.tests-path }}/xtest From 9cfc67322f470931c39f4b3b03792d743016c9b0 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 17 Apr 2026 08:04:29 -0400 Subject: [PATCH 17/21] fix: add OT_ROOT_KEY env and guard fromJson on empty outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Add OT_ROOT_KEY to ABAC test step env — the old km-check step set this via $GITHUB_ENV, but setup-test-environment only exposes it as an action output. 2. Guard fromJson() in setup-cli-tool checkout steps with fallback JSON ('{"sha":""}') to prevent JToken errors during Post cleanup when version-b/c/d outputs are empty strings. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 1 + xtest/setup-cli-tool/action.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 3ad813ba4..af47203c8 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -436,6 +436,7 @@ jobs: env: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" PLATFORM_TAG: ${{ matrix.platform-tag }} + OT_ROOT_KEY: ${{ steps.test-env.outputs.root-key }} PLATFORM_LOG_FILE: "../../${{ steps.run-platform.outputs.platform-log-file }}" KAS_ALPHA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-alpha-log-file }} KAS_BETA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-beta-log-file }} diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml index 29747fa4e..5bf6eff62 100644 --- a/xtest/setup-cli-tool/action.yaml +++ b/xtest/setup-cli-tool/action.yaml @@ -241,7 +241,7 @@ runs: with: path: ${{ steps.check-source.outputs.checkout-path-a }} persist-credentials: false - ref: ${{ fromJson(steps.resolve.outputs.version-a).sha }} + ref: ${{ fromJson(steps.resolve.outputs.version-a || '{"sha":""}').sha }} repository: ${{ steps.check-source.outputs.checkout-repo-a }} - name: checkout version b @@ -252,7 +252,7 @@ runs: with: path: ${{ steps.check-source.outputs.checkout-path-b }} persist-credentials: false - ref: ${{ fromJson(steps.resolve.outputs.version-b).sha }} + ref: ${{ fromJson(steps.resolve.outputs.version-b || '{"sha":""}').sha }} repository: ${{ steps.check-source.outputs.checkout-repo-b }} - name: checkout version c @@ -263,7 +263,7 @@ runs: with: path: ${{ steps.check-source.outputs.checkout-path-c }} persist-credentials: false - ref: ${{ fromJson(steps.resolve.outputs.version-c).sha }} + ref: ${{ fromJson(steps.resolve.outputs.version-c || '{"sha":""}').sha }} repository: ${{ steps.check-source.outputs.checkout-repo-c }} - name: checkout version d @@ -274,7 +274,7 @@ runs: with: path: ${{ steps.check-source.outputs.checkout-path-d }} persist-credentials: false - ref: ${{ fromJson(steps.resolve.outputs.version-d).sha }} + ref: ${{ fromJson(steps.resolve.outputs.version-d || '{"sha":""}').sha }} repository: ${{ steps.check-source.outputs.checkout-repo-d }} - name: symlink freshly checked-out platform sources From daf28e9083fb196fce39e5630d4e89b9bb7e755d Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 17 Apr 2026 09:34:54 -0400 Subject: [PATCH 18/21] feat(otdf-local): add xtest config YAML and local test runner Add `otdf-local xtest` subcommands for reproducible integration testing: - `xtest resolve` - resolves SDK versions via otdf-sdk-mgr, detects platform features, and generates a YAML config file - `xtest run` - executes the full test lifecycle from a config file: SDK installation, service startup, and pytest phase execution - `xtest show` - displays a human-readable config summary The YAML config captures everything needed to reproduce a CI test run locally: resolved SDK versions/SHAs, platform features, and test phase definitions. CI now generates per-matrix-cell configs and emits them in step summaries behind
tags for easy copy-paste reproduction. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 105 +++++++ otdf-local/src/otdf_local/cli.py | 2 + otdf-local/src/otdf_local/xtest/__init__.py | 10 + otdf-local/src/otdf_local/xtest/cli.py | 199 ++++++++++++ otdf-local/src/otdf_local/xtest/config.py | 236 +++++++++++++++ otdf-local/src/otdf_local/xtest/resolve.py | 189 ++++++++++++ otdf-local/src/otdf_local/xtest/runner.py | 316 ++++++++++++++++++++ otdf-local/tests/test_xtest_config.py | 222 ++++++++++++++ 8 files changed, 1279 insertions(+) create mode 100644 otdf-local/src/otdf_local/xtest/__init__.py create mode 100644 otdf-local/src/otdf_local/xtest/cli.py create mode 100644 otdf-local/src/otdf_local/xtest/config.py create mode 100644 otdf-local/src/otdf_local/xtest/resolve.py create mode 100644 otdf-local/src/otdf_local/xtest/runner.py create mode 100644 otdf-local/tests/test_xtest_config.py diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index af47203c8..02e52ccdf 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -133,6 +133,7 @@ jobs: sparse-checkout: | xtest/sdk otdf-sdk-mgr + otdf-local - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b with: python-version: "3.14" @@ -252,6 +253,70 @@ jobs: throw new Error('Errors detected in version resolution. Failing the run.'); } + - name: Generate xtest configs + id: xtest-configs + shell: bash + run: | + OTDF_LOCAL_DIR="${GITHUB_WORKSPACE}/otdf-sdk/otdf-local" + FOCUS_SDK="${FOCUS_SDK_INPUT:-all}" + OTDFCTL_SRC="${OTDFCTL_SOURCE_INPUT:-auto}" + PLATFORM_REFS="${PLATFORM_REF:-main}" + GO_REFS="${OTDFCTL_REF:-${DEFAULT_TAGS}}" + JS_REFS="${JS_REF:-${DEFAULT_TAGS}}" + JAVA_REFS="${JAVA_REF:-${DEFAULT_TAGS}}" + + mkdir -p xtest-configs + + # Generate one config per (platform-tag, sdk) matrix cell + PLATFORM_TAGS=$(echo "${{ steps.version-info.outputs.platform-tag-list }}" | jq -r '.[]') + for ptag in $PLATFORM_TAGS; do + for sdk in go java js; do + CONFIG_FILE="xtest-configs/${sdk}-${ptag}.yaml" + uv run --project "$OTDF_LOCAL_DIR" otdf-local xtest resolve \ + --platform-ref "$ptag" \ + --go-ref "$GO_REFS" \ + --js-ref "$JS_REFS" \ + --java-ref "$JAVA_REFS" \ + --focus-sdk "$FOCUS_SDK" \ + --otdfctl-source "$OTDFCTL_SRC" \ + --output "$CONFIG_FILE" 2>/dev/null || true + # Patch encrypt-sdk to match the matrix cell + if [ -f "$CONFIG_FILE" ]; then + sed -i "s/^encrypt-sdk:.*/encrypt-sdk: ${sdk}/" "$CONFIG_FILE" + fi + done + done + + # Emit a sample config to step summary for local reproduction + SAMPLE=$(ls xtest-configs/*.yaml 2>/dev/null | head -1) + if [ -n "$SAMPLE" ]; then + { + echo '
Reproduce locally (xtest config)' + echo '' + echo '```yaml' + cat "$SAMPLE" + echo '```' + echo '' + echo '**To reproduce locally:**' + echo '```bash' + echo '# Save the above YAML to a file, then:' + echo 'cd tests/otdf-local' + echo 'uv run otdf-local xtest run ../xtest-config.yaml' + echo '```' + echo '
' + } >> "$GITHUB_STEP_SUMMARY" + fi + env: + FOCUS_SDK_INPUT: "${{ inputs.focus-sdk }}" + OTDFCTL_SOURCE_INPUT: "${{ inputs.otdfctl-source }}" + + - name: Upload xtest configs + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: xtest-configs + path: xtest-configs/ + if-no-files-found: warn + xct: timeout-minutes: 60 runs-on: ubuntu-latest @@ -276,6 +341,24 @@ jobs: path: otdftests # use different name bc other repos might have tests directories persist-credentials: false + - name: Download xtest config + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: xtest-configs + path: xtest-configs + + - name: Set xtest config path + id: xtest-config + run: | + CONFIG="xtest-configs/${{ matrix.sdk }}-${{ matrix.platform-tag }}.yaml" + if [ -f "$CONFIG" ]; then + echo "config-file=$CONFIG" >> "$GITHUB_OUTPUT" + echo "Using xtest config: $CONFIG" + else + echo "config-file=" >> "$GITHUB_OUTPUT" + echo "No xtest config found for ${{ matrix.sdk }}-${{ matrix.platform-tag }}" + fi + - name: load extra keys from file id: load-extra-keys run: |- @@ -461,6 +544,28 @@ jobs: path: otdftests/xtest/test-results/audit-logs/*.log if-no-files-found: ignore + - name: Emit xtest config for reproduction + if: failure() && steps.xtest-config.outputs.config-file != '' + run: | + CONFIG="${{ steps.xtest-config.outputs.config-file }}" + { + echo '
Reproduce this failure locally' + echo '' + echo '```yaml' + cat "$CONFIG" + echo '```' + echo '' + echo '**To reproduce:**' + echo '```bash' + echo "cat > xtest-config.yaml <<'EOF'" + cat "$CONFIG" + echo 'EOF' + echo 'cd tests/otdf-local' + echo 'uv run otdf-local xtest run ../../xtest-config.yaml' + echo '```' + echo '
' + } >> "$GITHUB_STEP_SUMMARY" + publish-results: runs-on: ubuntu-latest needs: xct diff --git a/otdf-local/src/otdf_local/cli.py b/otdf-local/src/otdf_local/cli.py index 59fc8e098..a3bc51265 100644 --- a/otdf-local/src/otdf_local/cli.py +++ b/otdf-local/src/otdf_local/cli.py @@ -12,6 +12,7 @@ from otdf_local import __version__ from otdf_local.ci import ci_app +from otdf_local.xtest.cli import xtest_app from otdf_local.config.ports import Ports from otdf_local.config.settings import get_settings from otdf_local.health.waits import WaitTimeoutError, wait_for_health, wait_for_port @@ -45,6 +46,7 @@ ) app.add_typer(ci_app, name="ci") +app.add_typer(xtest_app, name="xtest") def _show_provision_error(result: ProvisionResult, target: str) -> None: diff --git a/otdf-local/src/otdf_local/xtest/__init__.py b/otdf-local/src/otdf_local/xtest/__init__.py new file mode 100644 index 000000000..03d10531f --- /dev/null +++ b/otdf-local/src/otdf_local/xtest/__init__.py @@ -0,0 +1,10 @@ +"""xtest configuration and execution subpackage.""" + +from otdf_local.xtest.config import Features, TestPhase, XtestConfig, XtestInputs + +__all__ = [ + "Features", + "TestPhase", + "XtestConfig", + "XtestInputs", +] diff --git a/otdf-local/src/otdf_local/xtest/cli.py b/otdf-local/src/otdf_local/xtest/cli.py new file mode 100644 index 000000000..084211d84 --- /dev/null +++ b/otdf-local/src/otdf_local/xtest/cli.py @@ -0,0 +1,199 @@ +"""Typer CLI subcommands for xtest configuration and execution.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Annotated + +import typer +from rich.table import Table + +from otdf_local.config.settings import get_settings +from otdf_local.utils.console import console, print_error, print_success +from otdf_local.xtest.config import XtestConfig, XtestInputs + +xtest_app = typer.Typer( + name="xtest", + help="Resolve, configure, and run xtest integration tests.", + no_args_is_help=True, +) + + +@xtest_app.command() +def resolve( + platform_ref: Annotated[ + str, + typer.Option( + "--platform-ref", help="Platform ref: branch, tag, SHA, 'latest', or 'lts'" + ), + ] = "main", + go_ref: Annotated[ + str, + typer.Option("--go-ref", help="Go/otdfctl ref"), + ] = "main", + js_ref: Annotated[ + str, + typer.Option("--js-ref", help="JS/web-sdk ref"), + ] = "main", + java_ref: Annotated[ + str, + typer.Option("--java-ref", help="Java SDK ref"), + ] = "main", + focus_sdk: Annotated[ + str, + typer.Option("--focus-sdk", help="SDK to focus on (go, js, java, all)"), + ] = "all", + otdfctl_source: Annotated[ + str, + typer.Option( + "--otdfctl-source", help="otdfctl source: auto, standalone, platform" + ), + ] = "auto", + output: Annotated[ + Path | None, + typer.Option("--output", "-o", help="Write config to file (default: stdout)"), + ] = None, +) -> None: + """Resolve SDK versions and generate an xtest configuration file. + + Calls otdf-sdk-mgr to resolve version refs to SHAs, detects platform features, + and outputs a YAML config suitable for `otdf-local xtest run`. + """ + from otdf_local.xtest.resolve import resolve_all + + if focus_sdk not in ("all", "go", "js", "java"): + print_error( + f"Invalid focus-sdk: {focus_sdk}. Must be one of: all, go, js, java" + ) + raise typer.Exit(1) + + inputs = XtestInputs( + platform_ref=platform_ref, + go_ref=go_ref, + js_ref=js_ref, + java_ref=java_ref, + focus_sdk=focus_sdk, + otdfctl_source=otdfctl_source, + ) + + settings = get_settings() + config = resolve_all(inputs, settings) + + yaml_output = config.to_yaml() + + if output: + config.to_yaml_file(output) + print_success(f"Config written to {output}") + else: + print(yaml_output, file=sys.stdout) + + +@xtest_app.command() +def run( + config_file: Annotated[ + Path, + typer.Argument(help="Path to xtest config YAML file"), + ], + phase: Annotated[ + str | None, + typer.Option( + "--phase", + "-p", + help="Run only this phase (helpers, legacy, standard, abac)", + ), + ] = None, + skip_services: Annotated[ + bool, + typer.Option("--skip-services", help="Assume services are already running"), + ] = False, + skip_install: Annotated[ + bool, + typer.Option("--skip-install", help="Assume SDKs are already installed"), + ] = False, +) -> None: + """Run xtest integration tests from a configuration file. + + Installs SDKs, starts services, and runs test phases as defined in the config. + + Example: + otdf-local xtest run xtest-config.yaml + otdf-local xtest run xtest-config.yaml --phase legacy --skip-services + """ + from otdf_local.xtest.runner import run_xtest + + if not config_file.exists(): + print_error(f"Config file not found: {config_file}") + raise typer.Exit(1) + + config = XtestConfig.from_yaml(config_file) + settings = get_settings() + + passed = run_xtest( + config=config, + settings=settings, + phase_name=phase, + skip_services=skip_services, + skip_install=skip_install, + ) + + if not passed: + raise typer.Exit(1) + + +@xtest_app.command() +def show( + config_file: Annotated[ + Path, + typer.Argument(help="Path to xtest config YAML file"), + ], +) -> None: + """Display a human-readable summary of an xtest configuration.""" + if not config_file.exists(): + print_error(f"Config file not found: {config_file}") + raise typer.Exit(1) + + config = XtestConfig.from_yaml(config_file) + + console.print(f"[bold]xtest config v{config.version}[/bold]") + console.print(f" Platform tag: {config.platform_tag}") + console.print(f" Encrypt SDK: {config.encrypt_sdk}") + console.print(f" Focus SDK: {config.inputs.focus_sdk}") + console.print() + + # Versions table + table = Table(title="Resolved Versions", show_header=True, header_style="bold") + table.add_column("SDK", width=10) + table.add_column("Tag", width=20) + table.add_column("SHA", width=10) + table.add_column("Type", width=10) + table.add_column("Error", width=30) + + for sdk, versions in config.resolved.items(): + for v in versions: + vtype = "head" if v.head else "release" if v.release else "?" + table.add_row( + sdk, + v.tag, + v.sha[:7] if v.sha else "", + vtype, + v.err or "", + ) + + console.print(table) + console.print() + + # Features + console.print("[bold]Features[/bold]") + console.print(f" EC TDF: {config.features.ec_tdf}") + console.print(f" Key Management: {config.features.key_management}") + console.print(f" Multi-KAS: {config.features.multikas}") + console.print() + + # Phases + console.print("[bold]Test Phases[/bold]") + for phase in config.phases: + reqs = f" (requires: {', '.join(phase.requires)})" if phase.requires else "" + met = config.check_phase_requirements(phase) + status = "[green]ready[/green]" if met else "[yellow]skipped[/yellow]" + console.print(f" {status} {phase.name}: {', '.join(phase.test_files)}{reqs}") diff --git a/otdf-local/src/otdf_local/xtest/config.py b/otdf-local/src/otdf_local/xtest/config.py new file mode 100644 index 000000000..e38041872 --- /dev/null +++ b/otdf-local/src/otdf_local/xtest/config.py @@ -0,0 +1,236 @@ +"""Pydantic models for xtest configuration.""" + +from __future__ import annotations + +import io +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, Field +from ruamel.yaml import YAML + +_yaml = YAML() +_yaml.default_flow_style = False +_yaml.width = 120 + + +class ResolvedVersion(BaseModel): + """A resolved SDK version, mirroring otdf-sdk-mgr ResolveResult.""" + + sdk: str + tag: str + sha: str = "" + alias: str = "" + head: bool = False + release: str = "" + source: str = "" + env: str = "" + err: str = "" + + +class TestPhase(BaseModel): + """A test phase definition.""" + + name: str + description: str = "" + test_files: list[str] = Field(default_factory=list) + pytest_args: list[str] = Field(default_factory=list) + requires: list[str] = Field(default_factory=list) + skip_on_dispatch: bool = False + env: dict[str, str] = Field(default_factory=dict) + + +class Features(BaseModel): + """Detected platform features relevant to test execution.""" + + ec_tdf: bool = True + key_management: bool = False + multikas: bool = True + + +class XtestInputs(BaseModel): + """Original input refs that were resolved.""" + + platform_ref: str = "main" + go_ref: str = "main" + js_ref: str = "main" + java_ref: str = "main" + focus_sdk: str = "all" + otdfctl_source: str = "auto" + + +class XtestConfig(BaseModel): + """Complete xtest configuration for a single test run.""" + + version: str = "1" + inputs: XtestInputs = Field(default_factory=XtestInputs) + resolved: dict[str, list[ResolvedVersion]] = Field(default_factory=dict) + platform_tag: str = "main" + encrypt_sdk: str = "go" + features: Features = Field(default_factory=Features) + phases: list[TestPhase] = Field(default_factory=lambda: list(DEFAULT_PHASES)) + + def to_yaml(self) -> str: + """Serialize to YAML string.""" + data = self._to_dict() + buf = io.StringIO() + _yaml.dump(data, buf) + return buf.getvalue() + + def to_yaml_file(self, path: Path) -> None: + """Write config to a YAML file.""" + data = self._to_dict() + with open(path, "w") as f: + _yaml.dump(data, f) + + def _to_dict(self) -> dict[str, Any]: + """Convert to a plain dict suitable for YAML serialization.""" + data: dict[str, Any] = { + "version": self.version, + "inputs": _strip_defaults(self.inputs.model_dump(), XtestInputs()), + "resolved": {}, + "platform-tag": self.platform_tag, + "encrypt-sdk": self.encrypt_sdk, + "features": _strip_defaults(self.features.model_dump(), Features()), + "phases": [], + } + for sdk, versions in self.resolved.items(): + data["resolved"][sdk] = [_strip_empty(v.model_dump()) for v in versions] + for phase in self.phases: + p: dict[str, Any] = {"name": phase.name} + if phase.description: + p["description"] = phase.description + if phase.test_files: + p["test-files"] = phase.test_files + if phase.pytest_args: + p["pytest-args"] = phase.pytest_args + if phase.requires: + p["requires"] = phase.requires + if phase.skip_on_dispatch: + p["skip-on-dispatch"] = True + if phase.env: + p["env"] = phase.env + data["phases"].append(p) + return data + + @classmethod + def from_yaml(cls, source: str | Path) -> XtestConfig: + """Parse config from a YAML string or file path.""" + if isinstance(source, Path): + with open(source) as f: + data = _yaml.load(f) + else: + data = _yaml.load(source) + return cls._from_dict(data) + + @classmethod + def _from_dict(cls, data: dict[str, Any]) -> XtestConfig: + """Build config from a parsed YAML dict.""" + inputs_data = data.get("inputs", {}) + inputs = XtestInputs( + platform_ref=inputs_data.get("platform-ref", "main"), + go_ref=inputs_data.get("go-ref", "main"), + js_ref=inputs_data.get("js-ref", "main"), + java_ref=inputs_data.get("java-ref", "main"), + focus_sdk=inputs_data.get("focus-sdk", "all"), + otdfctl_source=inputs_data.get("otdfctl-source", "auto"), + ) + + resolved: dict[str, list[ResolvedVersion]] = {} + for sdk, versions in data.get("resolved", {}).items(): + resolved[sdk] = [ResolvedVersion(**v) for v in versions] + + features_data = data.get("features", {}) + features = Features( + ec_tdf=features_data.get("ec-tdf", True), + key_management=features_data.get("key-management", False), + multikas=features_data.get("multikas", True), + ) + + phases = [] + for p in data.get("phases", []): + phases.append( + TestPhase( + name=p["name"], + description=p.get("description", ""), + test_files=p.get("test-files", []), + pytest_args=p.get("pytest-args", []), + requires=p.get("requires", []), + skip_on_dispatch=p.get("skip-on-dispatch", False), + env=p.get("env", {}), + ) + ) + + return cls( + version=data.get("version", "1"), + inputs=inputs, + resolved=resolved, + platform_tag=data.get("platform-tag", "main"), + encrypt_sdk=data.get("encrypt-sdk", "go"), + features=features, + phases=phases if phases else list(DEFAULT_PHASES), + ) + + def check_phase_requirements(self, phase: TestPhase) -> bool: + """Check if a phase's requirements are met by current features.""" + for req in phase.requires: + if req == "multikas" and not self.features.multikas: + return False + if req == "key-management" and not self.features.key_management: + return False + if req == "ec-tdf" and not self.features.ec_tdf: + return False + return True + + +def _strip_empty(d: dict[str, Any]) -> dict[str, Any]: + """Remove keys with empty/falsy values from a dict.""" + return {k: v for k, v in d.items() if v} + + +def _to_yaml_keys(d: dict[str, Any]) -> dict[str, Any]: + """Convert Python snake_case keys to YAML kebab-case keys.""" + return {k.replace("_", "-"): v for k, v in d.items()} + + +def _strip_defaults(d: dict[str, Any], defaults: BaseModel) -> dict[str, Any]: + """Remove keys that match the default model values, and convert to kebab-case.""" + default_dict = defaults.model_dump() + return _to_yaml_keys({k: v for k, v in d.items() if v != default_dict.get(k)}) + + +# Default test phases matching what xtest.yml runs +DEFAULT_PHASES: list[TestPhase] = [ + TestPhase( + name="helpers", + description="Validate xtest helper library", + test_files=["test_self.py", "test_audit_logs.py"], + skip_on_dispatch=True, + ), + TestPhase( + name="legacy", + description="Legacy decryption tests", + test_files=["test_legacy.py"], + pytest_args=["-n", "auto", "--dist", "worksteal"], + ), + TestPhase( + name="standard", + description="Standard TDF and policy tests", + test_files=["test_tdfs.py", "test_policytypes.py"], + pytest_args=["-n", "auto", "--dist", "loadscope"], + ), + TestPhase( + name="abac", + description="Attribute-based access control tests", + test_files=["test_abac.py"], + pytest_args=[ + "-n", + "auto", + "--dist", + "loadscope", + "--audit-log-dir", + "test-results/audit-logs", + ], + requires=["multikas"], + ), +] diff --git a/otdf-local/src/otdf_local/xtest/resolve.py b/otdf-local/src/otdf_local/xtest/resolve.py new file mode 100644 index 000000000..5f8d88f80 --- /dev/null +++ b/otdf-local/src/otdf_local/xtest/resolve.py @@ -0,0 +1,189 @@ +"""Version resolution via otdf-sdk-mgr subprocess calls.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +from otdf_local.config.features import PlatformFeatures +from otdf_local.config.settings import Settings +from otdf_local.utils.console import print_error, print_info, print_warning +from otdf_local.utils.yaml import get_nested, load_yaml +from otdf_local.xtest.config import ( + Features, + ResolvedVersion, + XtestConfig, + XtestInputs, +) + + +def _find_sdk_mgr_dir(settings: Settings) -> Path: + """Locate the otdf-sdk-mgr directory (sibling of otdf-local in the repo).""" + # Walk up from this file to find otdf-local root, then look for sibling + otdf_local_dir = Path(__file__).resolve().parent.parent.parent.parent + sdk_mgr = otdf_local_dir.parent / "otdf-sdk-mgr" + if sdk_mgr.is_dir(): + return sdk_mgr + # Try from xtest_root + sdk_mgr = settings.xtest_root.parent / "otdf-sdk-mgr" + if sdk_mgr.is_dir(): + return sdk_mgr + raise FileNotFoundError( + f"Could not find otdf-sdk-mgr directory. Checked: {otdf_local_dir.parent / 'otdf-sdk-mgr'}" + ) + + +def resolve_sdk_versions( + sdk: str, + refs: str, + sdk_mgr_dir: Path, + env_overrides: dict[str, str] | None = None, +) -> list[ResolvedVersion]: + """Call otdf-sdk-mgr versions resolve for a single SDK type. + + Args: + sdk: SDK type (platform, go, js, java) + refs: Space-separated version refs (e.g., "main latest") + sdk_mgr_dir: Path to otdf-sdk-mgr project + env_overrides: Extra environment variables (e.g., OTDFCTL_SOURCE) + + Returns: + List of ResolvedVersion objects + """ + import os + + ref_args = refs.strip().split() + cmd = [ + "uv", + "run", + "--project", + str(sdk_mgr_dir), + "otdf-sdk-mgr", + "versions", + "resolve", + sdk, + *ref_args, + ] + + env = dict(os.environ) + if env_overrides: + env.update(env_overrides) + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=str(sdk_mgr_dir), + env=env, + ) + + if result.returncode != 0: + return [ + ResolvedVersion( + sdk=sdk, + tag=refs, + err=result.stderr.strip() + or f"Process exited with code {result.returncode}", + ) + ] + + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as e: + return [ResolvedVersion(sdk=sdk, tag=refs, err=f"Invalid JSON output: {e}")] + + return [ResolvedVersion(**item) for item in data] + + +def detect_features(settings: Settings) -> Features: + """Detect platform features from the local platform config and version.""" + features = Features() + + # Try to detect from platform version + try: + pf = PlatformFeatures.detect(settings.platform_dir) + features.key_management = pf.supports("key_management") + features.ec_tdf = pf.supports("ecwrap") + # multikas is supported for main or version >= 0.4.x + features.multikas = pf.semver >= (0, 4, 0) + except Exception: + pass + + # Also check config file for explicit settings + try: + config = load_yaml(settings.platform_config) + ec_enabled = get_nested(config, "services.kas.preview.ec_tdf_enabled") + if ec_enabled is not None: + features.ec_tdf = bool(ec_enabled) + km_enabled = get_nested(config, "services.kas.preview.key_management") + if km_enabled is not None: + features.key_management = bool(km_enabled) + except Exception: + pass + + return features + + +def resolve_all(inputs: XtestInputs, settings: Settings) -> XtestConfig: + """Resolve all SDK versions and detect features, returning a complete config. + + Args: + inputs: The version refs and options to resolve + settings: otdf-local settings (for feature detection and path finding) + + Returns: + A fully populated XtestConfig + """ + sdk_mgr_dir = _find_sdk_mgr_dir(settings) + print_info(f"Using otdf-sdk-mgr at: {sdk_mgr_dir}") + + resolved: dict[str, list[ResolvedVersion]] = {} + has_errors = False + + # Resolve each SDK type + sdk_refs = { + "platform": inputs.platform_ref, + "go": inputs.go_ref, + "js": inputs.js_ref, + "java": inputs.java_ref, + } + + env_overrides: dict[str, str] = {} + if inputs.otdfctl_source == "platform": + env_overrides["OTDFCTL_SOURCE"] = "platform" + + for sdk, refs in sdk_refs.items(): + print_info(f"Resolving {sdk}: {refs}") + sdk_env = env_overrides if sdk == "go" else None + versions = resolve_sdk_versions(sdk, refs, sdk_mgr_dir, sdk_env) + resolved[sdk] = versions + + for v in versions: + if v.err: + print_error(f" Error resolving {sdk} {v.tag}: {v.err}") + has_errors = True + else: + head_marker = " (head)" if v.head else "" + print_info(f" {v.tag} -> {v.sha[:7]}{head_marker}") + + if has_errors: + print_warning("Some versions had errors; config may be incomplete") + + # Determine platform tag from resolved platform versions + platform_tags = [v.tag for v in resolved.get("platform", []) if not v.err] + platform_tag = platform_tags[0] if platform_tags else "main" + + # Detect features + try: + features = detect_features(settings) + except Exception: + features = Features() + + return XtestConfig( + inputs=inputs, + resolved=resolved, + platform_tag=platform_tag, + encrypt_sdk=inputs.focus_sdk if inputs.focus_sdk != "all" else "go", + features=features, + ) diff --git a/otdf-local/src/otdf_local/xtest/runner.py b/otdf-local/src/otdf_local/xtest/runner.py new file mode 100644 index 000000000..ec17062a3 --- /dev/null +++ b/otdf-local/src/otdf_local/xtest/runner.py @@ -0,0 +1,316 @@ +"""xtest runner - installs SDKs, manages services, runs test phases.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from otdf_local.config.settings import Settings +from otdf_local.utils.console import ( + console, + print_error, + print_info, + print_success, + print_warning, +) +from otdf_local.xtest.config import ResolvedVersion, TestPhase, XtestConfig +from otdf_local.xtest.resolve import _find_sdk_mgr_dir, detect_features + + +def install_sdks(config: XtestConfig, settings: Settings) -> bool: + """Install SDK CLIs based on resolved version info. + + For released versions, calls `otdf-sdk-mgr install artifact`. + For head versions, calls `otdf-sdk-mgr checkout` then `make`. + + Returns True if all installs succeeded. + """ + sdk_mgr_dir = _find_sdk_mgr_dir(settings) + sdk_base = settings.xtest_root / "sdk" + ok = True + + for sdk_type in ("go", "js", "java"): + versions = config.resolved.get(sdk_type, []) + for v in versions: + if v.err: + print_warning(f"Skipping {sdk_type} {v.tag}: has errors") + continue + + if v.head: + ok = _install_from_source(sdk_type, v, sdk_mgr_dir, sdk_base) and ok + elif v.release: + ok = _install_artifact(sdk_type, v, sdk_mgr_dir, sdk_base) and ok + else: + print_warning(f"Skipping {sdk_type} {v.tag}: neither head nor release") + + return ok + + +def _install_artifact( + sdk_type: str, + version: ResolvedVersion, + sdk_mgr_dir: Path, + sdk_base: Path, +) -> bool: + """Install a released SDK artifact via otdf-sdk-mgr.""" + print_info(f"Installing {sdk_type} {version.tag} from artifact...") + cmd = [ + "uv", + "run", + "--project", + str(sdk_mgr_dir), + "otdf-sdk-mgr", + "install", + "artifact", + "--sdk", + sdk_type, + "--version", + version.tag, + ] + if version.source: + cmd.extend(["--source", version.source]) + + result = subprocess.run(cmd, cwd=str(sdk_base / sdk_type)) + if result.returncode != 0: + print_error( + f"Failed to install {sdk_type} {version.tag} artifact, trying source..." + ) + return _install_from_source(sdk_type, version, sdk_mgr_dir, sdk_base) + + print_success(f"Installed {sdk_type} {version.tag}") + return True + + +def _install_from_source( + sdk_type: str, + version: ResolvedVersion, + sdk_mgr_dir: Path, + sdk_base: Path, +) -> bool: + """Checkout and build an SDK from source.""" + print_info(f"Building {sdk_type} {version.tag} from source...") + + # Checkout + cmd = [ + "uv", + "run", + "--project", + str(sdk_mgr_dir), + "otdf-sdk-mgr", + "checkout", + sdk_type, + version.tag, + ] + result = subprocess.run(cmd, cwd=str(sdk_base / sdk_type)) + if result.returncode != 0: + print_error(f"Failed to checkout {sdk_type} {version.tag}") + return False + + # Build + result = subprocess.run(["make"], cwd=str(sdk_base / sdk_type)) + if result.returncode != 0: + print_error(f"Failed to build {sdk_type} {version.tag}") + return False + + print_success(f"Built {sdk_type} {version.tag} from source") + return True + + +def run_phase( + phase: TestPhase, + config: XtestConfig, + settings: Settings, +) -> bool: + """Run a single test phase. + + Returns True if pytest exited successfully. + """ + xtest_dir = settings.xtest_root + + # Build pytest command + cmd = ["uv", "run", "pytest"] + cmd.extend(phase.pytest_args) + cmd.extend(["-ra", "-v"]) + + # Add focus/encrypt SDK flags + cmd.extend(["--sdks-encrypt", config.encrypt_sdk]) + if config.inputs.focus_sdk != "all": + cmd.extend(["--focus", config.inputs.focus_sdk]) + + # Add HTML report + report_name = f"{phase.name}-{config.encrypt_sdk}-{config.platform_tag}" + cmd.extend( + [ + "--html", + f"test-results/{report_name}.html", + "--self-contained-html", + ] + ) + + # Add test files + cmd.extend(phase.test_files) + + # Build environment + env = _build_phase_env(config, settings, phase) + + print_info(f"Running phase: {phase.name}") + print_info(f" Command: {' '.join(cmd)}") + + result = subprocess.run(cmd, cwd=str(xtest_dir), env=env) + + if result.returncode == 0: + print_success(f"Phase {phase.name} passed") + return True + else: + print_error(f"Phase {phase.name} failed (exit code {result.returncode})") + return False + + +def _build_phase_env( + config: XtestConfig, + settings: Settings, + phase: TestPhase, +) -> dict[str, str]: + """Build environment variables for a test phase.""" + env = dict(os.environ) + + # Core variables + env["PLATFORM_TAG"] = config.platform_tag + env["PLATFORM_DIR"] = str(settings.platform_dir.resolve()) + env["PLATFORMURL"] = settings.platform_url + env["ENCRYPT_SDK"] = config.encrypt_sdk + env["FOCUS_SDK"] = config.inputs.focus_sdk + + # Schema file + schema_file = settings.platform_dir / "sdk" / "schema" / "manifest.schema.json" + if schema_file.exists(): + env["SCHEMA_FILE"] = str(schema_file.resolve()) + else: + # Fallback to xtest-local manifest.schema.json + local_schema = settings.xtest_root / "manifest.schema.json" + if local_schema.exists(): + env["SCHEMA_FILE"] = "manifest.schema.json" + + # Log files + platform_log = settings.logs_dir / "platform.log" + if platform_log.exists(): + env["PLATFORM_LOG_FILE"] = str(platform_log.resolve()) + + kas_env_mapping = { + "alpha": "KAS_ALPHA_LOG_FILE", + "beta": "KAS_BETA_LOG_FILE", + "gamma": "KAS_GAMMA_LOG_FILE", + "delta": "KAS_DELTA_LOG_FILE", + "km1": "KAS_KM1_LOG_FILE", + "km2": "KAS_KM2_LOG_FILE", + } + for kas_name, env_var in kas_env_mapping.items(): + log_path = settings.get_kas_log_path(kas_name) + if log_path.exists(): + env[env_var] = str(log_path.resolve()) + + # Root key + from otdf_local.utils.yaml import get_nested, load_yaml + + try: + platform_config = load_yaml(settings.platform_config) + root_key = get_nested(platform_config, "services.kas.root_key") + if root_key: + env["OT_ROOT_KEY"] = root_key + except Exception: + pass + + # Phase-specific env overrides + env.update(phase.env) + + return env + + +def run_xtest( + config: XtestConfig, + settings: Settings, + phase_name: str | None = None, + skip_services: bool = False, + skip_install: bool = False, +) -> bool: + """Execute the full xtest lifecycle. + + Args: + config: Parsed xtest configuration + settings: otdf-local settings + phase_name: Run only this phase (None = all phases) + skip_services: Don't start/stop services + skip_install: Don't install SDKs + + Returns: + True if all phases passed + """ + # Ensure test-results directory exists + results_dir = settings.xtest_root / "test-results" + results_dir.mkdir(parents=True, exist_ok=True) + + # Step 1: Install SDKs + if not skip_install: + print_info("Installing SDK CLIs...") + if not install_sdks(config, settings): + print_warning("Some SDK installs failed; continuing with available SDKs") + + # Step 2: Start services + if not skip_services: + print_info("Starting services...") + from otdf_local.cli import up + + try: + up(services=None, no_provision=False) + except SystemExit as e: + if e.code != 0: + print_error("Failed to start services") + return False + + # Step 3: Re-detect features from running platform + try: + config.features = detect_features(settings) + print_info( + f"Features: ec-tdf={config.features.ec_tdf}, " + f"key-management={config.features.key_management}, " + f"multikas={config.features.multikas}" + ) + except Exception as e: + print_warning(f"Could not detect features: {e}") + + # Step 4: Run phases + phases = config.phases + if phase_name: + phases = [p for p in phases if p.name == phase_name] + if not phases: + valid = ", ".join(p.name for p in config.phases) + print_error(f"Unknown phase: {phase_name}. Valid phases: {valid}") + return False + + all_passed = True + results: list[tuple[str, bool, str]] = [] + + for phase in phases: + # Check requirements + if not config.check_phase_requirements(phase): + reason = f"unmet requirements: {', '.join(phase.requires)}" + print_warning(f"Skipping phase {phase.name}: {reason}") + results.append((phase.name, True, "skipped")) + continue + + passed = run_phase(phase, config, settings) + results.append((phase.name, passed, "passed" if passed else "FAILED")) + if not passed: + all_passed = False + + # Print summary + console.print() + console.print("[bold]Test Summary[/bold]") + for name, passed, status in results: + icon = "[green]PASS[/green]" if passed else "[red]FAIL[/red]" + if status == "skipped": + icon = "[yellow]SKIP[/yellow]" + console.print(f" {icon} {name}") + + return all_passed diff --git a/otdf-local/tests/test_xtest_config.py b/otdf-local/tests/test_xtest_config.py new file mode 100644 index 000000000..55daa80da --- /dev/null +++ b/otdf-local/tests/test_xtest_config.py @@ -0,0 +1,222 @@ +"""Tests for xtest config models: serialization, parsing, requirements.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +import pytest + +from otdf_local.xtest.config import ( + DEFAULT_PHASES, + Features, + ResolvedVersion, + TestPhase, + XtestConfig, + XtestInputs, +) + +SAMPLE_CONFIG_YAML = """\ +version: '1' +inputs: + platform-ref: main + go-ref: v0.29.0 + focus-sdk: go +resolved: + platform: + - sdk: platform + tag: main + sha: abc1234567890 + alias: main + head: true + go: + - sdk: go + tag: v0.29.0 + sha: def5678901234 + alias: latest + release: v0.29.0 + js: + - sdk: js + tag: '0.9.0' + sha: aaa1111222233 + alias: latest + release: sdk/0.9.0 + java: + - sdk: java + tag: v0.12.0 + sha: bbb2222333344 + alias: latest + release: v0.12.0 +platform-tag: main +encrypt-sdk: go +features: + ec-tdf: true + multikas: true +phases: +- name: legacy + test-files: + - test_legacy.py + pytest-args: + - -n + - auto +- name: abac + test-files: + - test_abac.py + requires: + - multikas +""" + + +class TestXtestConfigRoundTrip: + """Config can be serialized to YAML and parsed back identically.""" + + def test_round_trip_from_objects(self): + config = XtestConfig( + inputs=XtestInputs(platform_ref="main", go_ref="v0.29.0", focus_sdk="go"), + resolved={ + "platform": [ + ResolvedVersion(sdk="platform", tag="main", sha="abc123", head=True), + ], + "go": [ + ResolvedVersion(sdk="go", tag="v0.29.0", sha="def456", release="v0.29.0"), + ], + }, + platform_tag="main", + encrypt_sdk="go", + features=Features(ec_tdf=True, key_management=False, multikas=True), + phases=[ + TestPhase(name="legacy", test_files=["test_legacy.py"]), + TestPhase(name="abac", test_files=["test_abac.py"], requires=["multikas"]), + ], + ) + + yaml_str = config.to_yaml() + parsed = XtestConfig.from_yaml(yaml_str) + + assert parsed.version == config.version + assert parsed.platform_tag == config.platform_tag + assert parsed.encrypt_sdk == config.encrypt_sdk + assert parsed.inputs.focus_sdk == "go" + assert parsed.inputs.go_ref == "v0.29.0" + assert len(parsed.resolved["platform"]) == 1 + assert parsed.resolved["platform"][0].sha == "abc123" + assert parsed.resolved["go"][0].release == "v0.29.0" + assert parsed.features.ec_tdf is True + assert parsed.features.key_management is False + assert len(parsed.phases) == 2 + assert parsed.phases[1].requires == ["multikas"] + + def test_round_trip_file(self, tmp_path: Path): + config = XtestConfig( + resolved={"go": [ResolvedVersion(sdk="go", tag="main", sha="aaa111", head=True)]}, + ) + out_file = tmp_path / "config.yaml" + config.to_yaml_file(out_file) + parsed = XtestConfig.from_yaml(out_file) + assert parsed.resolved["go"][0].sha == "aaa111" + + def test_parse_sample_yaml(self): + config = XtestConfig.from_yaml(SAMPLE_CONFIG_YAML) + assert config.platform_tag == "main" + assert config.encrypt_sdk == "go" + assert config.inputs.go_ref == "v0.29.0" + assert config.inputs.focus_sdk == "go" + assert len(config.resolved["platform"]) == 1 + assert config.resolved["platform"][0].head is True + assert config.resolved["go"][0].release == "v0.29.0" + assert config.resolved["js"][0].tag == "0.9.0" + assert config.features.ec_tdf is True + assert config.features.multikas is True + assert len(config.phases) == 2 + assert config.phases[0].name == "legacy" + assert config.phases[1].requires == ["multikas"] + + +class TestDefaultPhases: + """Default phase definitions are valid and complete.""" + + def test_default_phases_exist(self): + assert len(DEFAULT_PHASES) == 4 + + def test_default_phase_names(self): + names = [p.name for p in DEFAULT_PHASES] + assert names == ["helpers", "legacy", "standard", "abac"] + + def test_abac_requires_multikas(self): + abac = next(p for p in DEFAULT_PHASES if p.name == "abac") + assert "multikas" in abac.requires + + def test_helpers_skip_on_dispatch(self): + helpers = next(p for p in DEFAULT_PHASES if p.name == "helpers") + assert helpers.skip_on_dispatch is True + + def test_all_phases_have_test_files(self): + for phase in DEFAULT_PHASES: + assert len(phase.test_files) > 0, f"Phase {phase.name} has no test files" + + +class TestFeatureRequirements: + """Phase requirement checking works correctly.""" + + def test_met_requirements(self): + config = XtestConfig(features=Features(multikas=True, ec_tdf=True)) + phase = TestPhase(name="abac", test_files=["test_abac.py"], requires=["multikas"]) + assert config.check_phase_requirements(phase) is True + + def test_unmet_requirements(self): + config = XtestConfig(features=Features(multikas=False)) + phase = TestPhase(name="abac", test_files=["test_abac.py"], requires=["multikas"]) + assert config.check_phase_requirements(phase) is False + + def test_no_requirements(self): + config = XtestConfig(features=Features()) + phase = TestPhase(name="legacy", test_files=["test_legacy.py"]) + assert config.check_phase_requirements(phase) is True + + def test_key_management_requirement(self): + config = XtestConfig(features=Features(key_management=False)) + phase = TestPhase(name="km", test_files=["test.py"], requires=["key-management"]) + assert config.check_phase_requirements(phase) is False + + config.features.key_management = True + assert config.check_phase_requirements(phase) is True + + def test_ec_tdf_requirement(self): + config = XtestConfig(features=Features(ec_tdf=False)) + phase = TestPhase(name="ec", test_files=["test.py"], requires=["ec-tdf"]) + assert config.check_phase_requirements(phase) is False + + +class TestStripDefaults: + """YAML output omits fields that match defaults for cleaner output.""" + + def test_default_inputs_stripped(self): + config = XtestConfig() + yaml_str = config.to_yaml() + # Default inputs should be stripped (empty dict or missing keys) + parsed = XtestConfig.from_yaml(yaml_str) + assert parsed.inputs.platform_ref == "main" + assert parsed.inputs.focus_sdk == "all" + + def test_non_default_inputs_preserved(self): + config = XtestConfig( + inputs=XtestInputs(focus_sdk="go", go_ref="v1.0.0"), + ) + yaml_str = config.to_yaml() + assert "focus-sdk: go" in yaml_str + assert "go-ref: v1.0.0" in yaml_str + + +class TestResolvedVersionErrors: + """Error versions are preserved through serialization.""" + + def test_error_version_round_trip(self): + config = XtestConfig( + resolved={ + "go": [ResolvedVersion(sdk="go", tag="bad-ref", err="Not found")], + }, + ) + yaml_str = config.to_yaml() + parsed = XtestConfig.from_yaml(yaml_str) + assert parsed.resolved["go"][0].err == "Not found" + assert parsed.resolved["go"][0].sha == "" From 1084033c9eb0a190d29bb260a849117293db4d0e Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 17 Apr 2026 09:59:44 -0400 Subject: [PATCH 19/21] fix(xtest): fix CI failures in xtest config generation - Pass platform-tag-list JSON via env var to avoid bash quote nesting issue with jq parsing - Remove unused imports (tempfile, pytest) flagged by ruff Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 3 ++- otdf-local/tests/test_xtest_config.py | 31 ++++++++++++++++++--------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 02e52ccdf..2922e3b8e 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -268,7 +268,7 @@ jobs: mkdir -p xtest-configs # Generate one config per (platform-tag, sdk) matrix cell - PLATFORM_TAGS=$(echo "${{ steps.version-info.outputs.platform-tag-list }}" | jq -r '.[]') + PLATFORM_TAGS=$(echo "$PLATFORM_TAG_LIST_JSON" | jq -r '.[]') for ptag in $PLATFORM_TAGS; do for sdk in go java js; do CONFIG_FILE="xtest-configs/${sdk}-${ptag}.yaml" @@ -309,6 +309,7 @@ jobs: env: FOCUS_SDK_INPUT: "${{ inputs.focus-sdk }}" OTDFCTL_SOURCE_INPUT: "${{ inputs.otdfctl-source }}" + PLATFORM_TAG_LIST_JSON: "${{ steps.version-info.outputs.platform-tag-list }}" - name: Upload xtest configs uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 diff --git a/otdf-local/tests/test_xtest_config.py b/otdf-local/tests/test_xtest_config.py index 55daa80da..85677c126 100644 --- a/otdf-local/tests/test_xtest_config.py +++ b/otdf-local/tests/test_xtest_config.py @@ -2,11 +2,8 @@ from __future__ import annotations -import tempfile from pathlib import Path -import pytest - from otdf_local.xtest.config import ( DEFAULT_PHASES, Features, @@ -75,10 +72,14 @@ def test_round_trip_from_objects(self): inputs=XtestInputs(platform_ref="main", go_ref="v0.29.0", focus_sdk="go"), resolved={ "platform": [ - ResolvedVersion(sdk="platform", tag="main", sha="abc123", head=True), + ResolvedVersion( + sdk="platform", tag="main", sha="abc123", head=True + ), ], "go": [ - ResolvedVersion(sdk="go", tag="v0.29.0", sha="def456", release="v0.29.0"), + ResolvedVersion( + sdk="go", tag="v0.29.0", sha="def456", release="v0.29.0" + ), ], }, platform_tag="main", @@ -86,7 +87,9 @@ def test_round_trip_from_objects(self): features=Features(ec_tdf=True, key_management=False, multikas=True), phases=[ TestPhase(name="legacy", test_files=["test_legacy.py"]), - TestPhase(name="abac", test_files=["test_abac.py"], requires=["multikas"]), + TestPhase( + name="abac", test_files=["test_abac.py"], requires=["multikas"] + ), ], ) @@ -108,7 +111,9 @@ def test_round_trip_from_objects(self): def test_round_trip_file(self, tmp_path: Path): config = XtestConfig( - resolved={"go": [ResolvedVersion(sdk="go", tag="main", sha="aaa111", head=True)]}, + resolved={ + "go": [ResolvedVersion(sdk="go", tag="main", sha="aaa111", head=True)] + }, ) out_file = tmp_path / "config.yaml" config.to_yaml_file(out_file) @@ -160,12 +165,16 @@ class TestFeatureRequirements: def test_met_requirements(self): config = XtestConfig(features=Features(multikas=True, ec_tdf=True)) - phase = TestPhase(name="abac", test_files=["test_abac.py"], requires=["multikas"]) + phase = TestPhase( + name="abac", test_files=["test_abac.py"], requires=["multikas"] + ) assert config.check_phase_requirements(phase) is True def test_unmet_requirements(self): config = XtestConfig(features=Features(multikas=False)) - phase = TestPhase(name="abac", test_files=["test_abac.py"], requires=["multikas"]) + phase = TestPhase( + name="abac", test_files=["test_abac.py"], requires=["multikas"] + ) assert config.check_phase_requirements(phase) is False def test_no_requirements(self): @@ -175,7 +184,9 @@ def test_no_requirements(self): def test_key_management_requirement(self): config = XtestConfig(features=Features(key_management=False)) - phase = TestPhase(name="km", test_files=["test.py"], requires=["key-management"]) + phase = TestPhase( + name="km", test_files=["test.py"], requires=["key-management"] + ) assert config.check_phase_requirements(phase) is False config.features.key_management = True From 9cbdd5ab05f4eee683fae602de5d0e0c556bc73c Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 17 Apr 2026 10:13:11 -0400 Subject: [PATCH 20/21] fix(xtest): make config generation non-blocking and add debug output - Add continue-on-error and set +e so config generation doesn't block the rest of the workflow - Remove stderr suppression to surface actual errors - Add echo statements for debugging - Make artifact download tolerant of missing configs Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/xtest.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 2922e3b8e..59350831f 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -255,12 +255,13 @@ jobs: - name: Generate xtest configs id: xtest-configs + continue-on-error: true shell: bash run: | + set +e # Don't abort on errors; this step is supplementary OTDF_LOCAL_DIR="${GITHUB_WORKSPACE}/otdf-sdk/otdf-local" FOCUS_SDK="${FOCUS_SDK_INPUT:-all}" OTDFCTL_SRC="${OTDFCTL_SOURCE_INPUT:-auto}" - PLATFORM_REFS="${PLATFORM_REF:-main}" GO_REFS="${OTDFCTL_REF:-${DEFAULT_TAGS}}" JS_REFS="${JS_REF:-${DEFAULT_TAGS}}" JAVA_REFS="${JAVA_REF:-${DEFAULT_TAGS}}" @@ -272,6 +273,7 @@ jobs: for ptag in $PLATFORM_TAGS; do for sdk in go java js; do CONFIG_FILE="xtest-configs/${sdk}-${ptag}.yaml" + echo "Generating config: $CONFIG_FILE" uv run --project "$OTDF_LOCAL_DIR" otdf-local xtest resolve \ --platform-ref "$ptag" \ --go-ref "$GO_REFS" \ @@ -279,7 +281,7 @@ jobs: --java-ref "$JAVA_REFS" \ --focus-sdk "$FOCUS_SDK" \ --otdfctl-source "$OTDFCTL_SRC" \ - --output "$CONFIG_FILE" 2>/dev/null || true + --output "$CONFIG_FILE" || echo "Warning: config generation failed for ${sdk}-${ptag}" # Patch encrypt-sdk to match the matrix cell if [ -f "$CONFIG_FILE" ]; then sed -i "s/^encrypt-sdk:.*/encrypt-sdk: ${sdk}/" "$CONFIG_FILE" @@ -305,6 +307,8 @@ jobs: echo '```' echo '
' } >> "$GITHUB_STEP_SUMMARY" + else + echo "Warning: No xtest configs were generated" fi env: FOCUS_SDK_INPUT: "${{ inputs.focus-sdk }}" @@ -312,6 +316,7 @@ jobs: PLATFORM_TAG_LIST_JSON: "${{ steps.version-info.outputs.platform-tag-list }}" - name: Upload xtest configs + if: steps.xtest-configs.outcome == 'success' uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: xtest-configs @@ -343,6 +348,7 @@ jobs: persist-credentials: false - name: Download xtest config + continue-on-error: true uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: name: xtest-configs From 6e13659e7196263d080c1ec9d943e406fa734ad5 Mon Sep 17 00:00:00 2001 From: Dave Mihalcik Date: Fri, 17 Apr 2026 13:25:04 -0400 Subject: [PATCH 21/21] fix(xtest): allow resolve to work without local platform checkout Make Settings optional in resolve flow so `otdf-local xtest resolve` works without a platform directory present. Feature detection is skipped when platform config isn't available. Co-Authored-By: Claude Opus 4.6 (1M context) --- otdf-local/src/otdf_local/xtest/cli.py | 5 +++- otdf-local/src/otdf_local/xtest/resolve.py | 28 ++++++++++++---------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/otdf-local/src/otdf_local/xtest/cli.py b/otdf-local/src/otdf_local/xtest/cli.py index 084211d84..eca67266f 100644 --- a/otdf-local/src/otdf_local/xtest/cli.py +++ b/otdf-local/src/otdf_local/xtest/cli.py @@ -77,7 +77,10 @@ def resolve( otdfctl_source=otdfctl_source, ) - settings = get_settings() + try: + settings = get_settings() + except (FileNotFoundError, Exception): + settings = None config = resolve_all(inputs, settings) yaml_output = config.to_yaml() diff --git a/otdf-local/src/otdf_local/xtest/resolve.py b/otdf-local/src/otdf_local/xtest/resolve.py index 5f8d88f80..7eb6e1844 100644 --- a/otdf-local/src/otdf_local/xtest/resolve.py +++ b/otdf-local/src/otdf_local/xtest/resolve.py @@ -18,17 +18,18 @@ ) -def _find_sdk_mgr_dir(settings: Settings) -> Path: +def _find_sdk_mgr_dir(settings: Settings | None = None) -> Path: """Locate the otdf-sdk-mgr directory (sibling of otdf-local in the repo).""" # Walk up from this file to find otdf-local root, then look for sibling otdf_local_dir = Path(__file__).resolve().parent.parent.parent.parent sdk_mgr = otdf_local_dir.parent / "otdf-sdk-mgr" if sdk_mgr.is_dir(): return sdk_mgr - # Try from xtest_root - sdk_mgr = settings.xtest_root.parent / "otdf-sdk-mgr" - if sdk_mgr.is_dir(): - return sdk_mgr + # Try from xtest_root if settings available + if settings is not None: + sdk_mgr = settings.xtest_root.parent / "otdf-sdk-mgr" + if sdk_mgr.is_dir(): + return sdk_mgr raise FileNotFoundError( f"Could not find otdf-sdk-mgr directory. Checked: {otdf_local_dir.parent / 'otdf-sdk-mgr'}" ) @@ -125,12 +126,13 @@ def detect_features(settings: Settings) -> Features: return features -def resolve_all(inputs: XtestInputs, settings: Settings) -> XtestConfig: +def resolve_all(inputs: XtestInputs, settings: Settings | None = None) -> XtestConfig: """Resolve all SDK versions and detect features, returning a complete config. Args: inputs: The version refs and options to resolve - settings: otdf-local settings (for feature detection and path finding) + settings: otdf-local settings (for feature detection and path finding). + Optional - if not provided, feature detection is skipped. Returns: A fully populated XtestConfig @@ -174,11 +176,13 @@ def resolve_all(inputs: XtestInputs, settings: Settings) -> XtestConfig: platform_tags = [v.tag for v in resolved.get("platform", []) if not v.err] platform_tag = platform_tags[0] if platform_tags else "main" - # Detect features - try: - features = detect_features(settings) - except Exception: - features = Features() + # Detect features (only if platform dir is available) + features = Features() + if settings is not None: + try: + features = detect_features(settings) + except Exception: + pass return XtestConfig( inputs=inputs,