diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml
index 773448892..af47203c8 100644
--- a/.github/workflows/xtest.yml
+++ b/.github/workflows/xtest.yml
@@ -28,6 +28,11 @@ on:
type: string
default: all
description: "SDK to focus on (go, js, java, all)"
+ otdfctl-source:
+ required: false
+ type: string
+ default: auto
+ description: "otdfctl source: 'auto' (standalone for releases, detect platform for head builds), 'standalone', or 'platform'"
workflow_call:
inputs:
platform-ref:
@@ -50,6 +55,10 @@ on:
required: false
type: string
default: all
+ otdfctl-source:
+ required: false
+ type: string
+ default: auto
schedule:
- cron: "30 6 * * *" # 0630 UTC
- cron: "0 5 * * 1,3" # 500 UTC (Monday, Wednesday)
@@ -78,6 +87,14 @@ jobs:
JS_REF: "${{ inputs.js-ref }}"
OTDFCTL_REF: "${{ inputs.otdfctl-ref }}"
JAVA_REF: "${{ inputs.java-ref }}"
+ # When explicitly set to 'platform', tells the Go resolver to resolve
+ # against opentdf/platform tags instead of the standalone otdfctl repo.
+ # In 'auto' mode, releases resolve from standalone; the detect-otdfctl
+ # step later probes the platform checkout for an embedded otdfctl
+ # directory, and setup-cli-tool acts on this only for versions that need
+ # a source checkout (head or artifact-install failure), reusing the
+ # platform checkout only when the resolved SHA matches.
+ OTDFCTL_SOURCE: "${{ inputs.otdfctl-source == 'platform' && 'platform' || '' }}"
steps:
- name: Validate focus-sdk input
if: ${{ inputs.focus-sdk != '' }}
@@ -170,7 +187,7 @@ jobs:
core.summary.addHeading('Versions under Test', 3);
- function artifactLink(sdkType, tag, release, head) {
+ function artifactLink(sdkType, tag, release, head, source) {
if (head || !release) return '';
const v = tag.replace(/^v/, '');
if (sdkType === 'js') {
@@ -182,7 +199,10 @@ jobs:
return `Maven Central`;
}
if (sdkType === 'go') {
- const url = `https://pkg.go.dev/github.com/opentdf/otdfctl@${encodeURIComponent(tag)}`;
+ const modulePath = source === 'platform'
+ ? `github.com/opentdf/platform/otdfctl`
+ : `github.com/opentdf/otdfctl`;
+ const url = `https://pkg.go.dev/${modulePath}@${encodeURIComponent(tag)}`;
return `pkg.go.dev`;
}
return '';
@@ -198,14 +218,15 @@ jobs:
const tagToSha = {};
const headTags = [];
- for (const { tag, head, sha, alias, err, release } of refInfo) {
- const sdkRepoUrl = `https://github.com/opentdf/${encodeURIComponent(sdkType == 'js' ? 'web-sdk' : sdkType == 'go' ? 'otdfctl' : sdkType == 'java' ? 'java-sdk' : sdkType)}`;
+ for (const { tag, head, sha, alias, err, release, source } of refInfo) {
+ const goRepoName = source === 'platform' ? 'platform' : 'otdfctl';
+ const sdkRepoUrl = `https://github.com/opentdf/${encodeURIComponent(sdkType == 'js' ? 'web-sdk' : sdkType == 'go' ? goRepoName : sdkType == 'java' ? 'java-sdk' : sdkType)}`;
const sdkLink = `${htmlEscape(sdkType)}`;
const commitLink = sha ? `${htmlEscape(sha.substring(0, 7))}` : ' . ';
const tagLink = (release && tag)
? `${htmlEscape(tag)}`
: tag ? htmlEscape(tag) : 'N/A';
- const artifactCell = artifactLink(sdkType, tag, release, head);
+ const artifactCell = artifactLink(sdkType, tag, release, head, source);
table.push([sdkLink, tagLink, commitLink, alias || 'N/A', artifactCell || 'N/A', err || 'N/A']);
if (err) {
errorCount += 1;
@@ -290,191 +311,56 @@ jobs:
with:
node-version: "22.x"
- ######### CHECKOUT JS CLI #############
- - name: Configure js-sdk
- id: configure-js
- uses: ./otdftests/xtest/setup-cli-tool
+ ######## SETUP TEST ENVIRONMENT #############
+ - name: Setup test environment
+ id: test-env
+ uses: ./otdftests/xtest/setup-test-environment
+ with:
+ platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }}
+ platform-tag: ${{ matrix.platform-tag }}
+ otdfctl-source-input: ${{ inputs.otdfctl-source }}
+
+ ######## SETUP SDK CLIENTS #############
+ - name: Setup JS SDK client
+ id: setup-js
+ uses: ./otdftests/xtest/setup-sdk-clients
with:
- path: otdftests/xtest/sdk
sdk: js
version-info: "${{ needs.resolve-versions.outputs.js }}"
-
- - name: Cache npm
- if: fromJson(steps.configure-js.outputs.heads)[0] != null
- uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2
+ platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }}
+ platform-heads: ${{ needs.resolve-versions.outputs.heads }}
+ platform-tag: ${{ matrix.platform-tag }}
+ platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }}
+
+ - name: Setup Go SDK client (otdfctl)
+ id: setup-go
+ uses: ./otdftests/xtest/setup-sdk-clients
with:
- path: ~/.npm
- key: npm-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/js/src/**/package-lock.json') }}
- restore-keys: |
- npm-${{ runner.os }}-
-
- ######## SETUP THE JS CLI #############
- - name: build and setup the web-sdk cli
- id: build-web-sdk
- if: fromJson(steps.configure-js.outputs.heads)[0] != null
- run: |
- make
- working-directory: otdftests/xtest/sdk/js
-
- ######## CHECKOUT GO CLI #############
- - name: Configure otdfctl
- id: configure-go
- uses: ./otdftests/xtest/setup-cli-tool
- with:
- path: otdftests/xtest/sdk
sdk: go
version-info: "${{ needs.resolve-versions.outputs.go }}"
-
- - name: Cache Go modules
- if: fromJson(steps.configure-go.outputs.heads)[0] != null
- uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2
+ platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }}
+ platform-heads: ${{ needs.resolve-versions.outputs.heads }}
+ platform-tag: ${{ matrix.platform-tag }}
+ platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }}
+ otdfctl-source: ${{ steps.test-env.outputs.otdfctl-source }}
+ otdfctl-dir: ${{ steps.test-env.outputs.otdfctl-dir }}
+ otdfctl-sha: ${{ steps.test-env.outputs.otdfctl-sha }}
+ focus-sdk: ${{ inputs.focus-sdk || 'all' }}
+
+ - name: Setup Java SDK client
+ id: setup-java
+ uses: ./otdftests/xtest/setup-sdk-clients
with:
- path: |
- ~/.cache/go-build
- ~/go/pkg/mod
- key: go-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/go/src/*/go.sum') }}
- restore-keys: |
- go-${{ runner.os }}-
-
- - name: Resolve otdfctl heads
- id: resolve-otdfctl-heads
- if: fromJson(steps.configure-go.outputs.heads)[0] != null
- run: |-
- echo "OTDFCTL_HEADS=$OTDFCTL_HEADS" >> "$GITHUB_ENV"
- env:
- OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }}
-
- - name: Replace otdfctl go.mod packages, but only at head version of platform
- if: fromJson(steps.configure-go.outputs.heads)[0] != null && env.FOCUS_SDK == 'go' && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag)
- env:
- PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }}
- run: |-
- echo "Replacing go.mod packages..."
- PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}"
- OTDFCTL_DIR_ABS="$(pwd)/otdftests/xtest/sdk/go/src/"
- echo "PLATFORM_DIR_ABS: $PLATFORM_DIR_ABS"
- echo "OTDFCTL_DIR_ABS: $OTDFCTL_DIR_ABS"
- for head in $(echo "${OTDFCTL_HEADS}" | jq -r '.[]'); do
- echo "Processing head: $head"
- cd "${OTDFCTL_DIR_ABS}/$head"
- for m in lib/fixtures lib/ocrypto protocol/go sdk; do
- go mod edit -replace "github.com/opentdf/platform/$m=${PLATFORM_DIR_ABS}/$m"
- done
- go mod tidy
- done
-
- ######## SETUP THE GO CLI #############
- - name: Prepare go cli
- if: fromJson(steps.configure-go.outputs.heads)[0] != null
- run: |-
- make
- working-directory: otdftests/xtest/sdk/go
-
- ####### CHECKOUT JAVA SDK ##############
-
- - name: Configure java-sdk
- id: configure-java
- uses: ./otdftests/xtest/setup-cli-tool
- with:
- path: otdftests/xtest/sdk
sdk: java
version-info: "${{ needs.resolve-versions.outputs.java }}"
-
- - name: Cache Maven repository
- if: fromJson(steps.configure-java.outputs.heads)[0] != null
- uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2
- with:
- path: ~/.m2/repository
- key: maven-${{ runner.os }}-${{ hashFiles('otdftests/xtest/sdk/java/src/**/pom.xml') }}
- restore-keys: |
- maven-${{ runner.os }}-
-
- - name: pre-release protocol buffers for java-sdk
- if: >-
- fromJson(steps.configure-java.outputs.heads)[0] != null
- && (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java')
- && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag)
- run: |-
- echo "Replacing .env files for java-sdk..."
- echo "Platform tag: $platform_tag"
- echo "Java version info: $java_version_info"
- for row in $(echo "$java_version_info" | jq -c '.[]'); do
- TAG=$(echo "$row" | jq -r '.tag')
- HEAD=$(echo "$row" | jq -r '.head')
- if [[ "$HEAD" == "true" ]]; then
- echo "Creating .env file for tag: [$TAG]; pointing to platform ref [$platform_tag]"
- echo "PLATFORM_BRANCH=$platform_ref" > "otdftests/xtest/sdk/java/${TAG}.env"
- else
- echo "Skipping .env file creation for release version [$TAG]"
- fi
- done
- env:
- java_version_info: ${{ needs.resolve-versions.outputs.java }}
- platform_ref: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-to-sha)[matrix.platform-tag] }}
- platform_tag: ${{ matrix.platform-tag }}
-
- ####### SETUP JAVA CLI ##############
- - name: Prepare java cli
- if: fromJson(steps.configure-java.outputs.heads)[0] != null
- run: |
- make
- working-directory: otdftests/xtest/sdk/java
- env:
- BUF_INPUT_HTTPS_USERNAME: opentdf-bot
- BUF_INPUT_HTTPS_PASSWORD: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }}
-
- ######## Configure test environment #############
- - name: Lookup current platform version
- id: platform-version
- run: |-
- if ! go run ./service version; then
- # NOTE: the version command was added in 0.4.37
- echo "Error: Unable to get platform version; defaulting to tag: [$PLATFORM_TAG]"
- echo "PLATFORM_VERSION=$PLATFORM_TAG" >> "$GITHUB_ENV"
- exit
- fi
- # Older version commands output version to stderr; newer versions output to stdout
- PLATFORM_VERSION=$(go run ./service version 2>&1)
- echo "PLATFORM_VERSION=$PLATFORM_VERSION" >> "$GITHUB_ENV"
- echo "## Platform version output: [$PLATFORM_VERSION]"
- working-directory: ${{ steps.run-platform.outputs.platform-working-dir }}
- env:
- PLATFORM_TAG: ${{ matrix.platform-tag }}
-
- - name: Check key management support and prepare root key
- id: km-check
- run: |-
- OT_CONFIG_FILE="$(pwd)/opentdf.yaml"
- echo "OT_CONFIG_FILE=$OT_CONFIG_FILE" >> "$GITHUB_ENV"
- # Determine if the config declares the key_management field
- km_value=$(yq e '.services.kas.preview.key_management' "$OT_CONFIG_FILE" 2>/dev/null || echo "null")
- case "$km_value" in
- true|false)
- echo "KEY_MANAGEMENT_SUPPORTED=true" >> "$GITHUB_ENV"
- echo "supported=true" >> "$GITHUB_OUTPUT"
- ;;
- *)
- echo "KEY_MANAGEMENT_SUPPORTED=false" >> "$GITHUB_ENV"
- echo "supported=false" >> "$GITHUB_OUTPUT"
- ;;
- esac
- # Prepare a root key for use by additional KAS instances
- existing_root_key=$(yq e '.services.kas.root_key' "$OT_CONFIG_FILE" 2>/dev/null || echo "")
- if [ -n "$existing_root_key" ] && [ "$existing_root_key" != "null" ]; then
- echo "Using existing root key from config"
- echo "OT_ROOT_KEY=$existing_root_key" >> "$GITHUB_ENV"
- echo "root_key=$existing_root_key" >> "$GITHUB_OUTPUT"
- else
- echo "Generating a new root key for additional KAS"
- gen_root_key=$(openssl rand -hex 32)
- echo "OT_ROOT_KEY=$gen_root_key" >> "$GITHUB_ENV"
- echo "root_key=$gen_root_key" >> "$GITHUB_OUTPUT"
- fi
- working-directory: ${{ steps.run-platform.outputs.platform-working-dir }}
-
- - name: Install test dependencies
- run: uv sync
- working-directory: otdftests/xtest
+ platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }}
+ platform-heads: ${{ needs.resolve-versions.outputs.heads }}
+ platform-tag: ${{ matrix.platform-tag }}
+ platform-tag-to-sha: ${{ needs.resolve-versions.outputs.platform-tag-to-sha }}
+ focus-sdk: ${{ inputs.focus-sdk || 'all' }}
+ pat-opentdf: ${{ secrets.PERSONAL_ACCESS_TOKEN_OPENTDF }}
+
+ ######## VALIDATE HELPERS #############
- name: Validate xtest helper library (tests of the test harness and its utilities)
if: ${{ !inputs }}
run: |-
@@ -523,92 +409,17 @@ jobs:
######## ATTRIBUTE BASED CONFIGURATION #############
- - name: Does platform support multikas?
- id: multikas
- run: |-
- if [[ $PLATFORM_TAG == main ]]; then
- echo "Main supports multikas"
- echo "supported=true" >> "$GITHUB_OUTPUT"
- elif awk -F. '{ if ($1 > 0 || ($1 == 0 && $2 > 4)) exit 0; else exit 1; }' <<< "${PLATFORM_VERSION#v}"; then
- echo "Selected version [$PLATFORM_VERSION] supports multikas"
- echo "supported=true" >> "$GITHUB_OUTPUT"
- else
- echo "At tag [$PLATFORM_TAG], [$PLATFORM_VERSION] probably does not support multikas"
- echo "supported=false" >> "$GITHUB_OUTPUT"
- fi
- env:
- PLATFORM_TAG: ${{ matrix.platform-tag }}
-
- - name: Start additional kas
- id: kas-alpha
- if: ${{ steps.multikas.outputs.supported == 'true' }}
- uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix
- with:
- ec-tdf-enabled: true
- kas-name: alpha
- kas-port: 8181
- log-type: json
- root-key: ${{ steps.km-check.outputs.root_key }}
-
- - name: Start additional kas
- id: kas-beta
- if: ${{ steps.multikas.outputs.supported == 'true' }}
- uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix
- with:
- ec-tdf-enabled: true
- kas-name: beta
- kas-port: 8282
- log-type: json
- root-key: ${{ steps.km-check.outputs.root_key }}
-
- - name: Start additional kas
- id: kas-gamma
- if: ${{ steps.multikas.outputs.supported == 'true' }}
- uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix
- with:
- ec-tdf-enabled: true
- kas-name: gamma
- kas-port: 8383
- log-type: json
- root-key: ${{ steps.km-check.outputs.root_key }}
-
- - name: Start additional kas
- id: kas-delta
- if: ${{ steps.multikas.outputs.supported == 'true' }}
- uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix
+ - name: Start KAS instances for ABAC tests
+ id: kas-instances
+ if: ${{ steps.test-env.outputs.multikas-supported == 'true' }}
+ uses: ./otdftests/xtest/setup-kas-instances
with:
- ec-tdf-enabled: true
- kas-port: 8484
- kas-name: delta
- log-type: json
- root-key: ${{ steps.km-check.outputs.root_key }}
-
- - name: Start additional KM kas (km1)
- id: kas-km1
- if: ${{ steps.multikas.outputs.supported == 'true' }}
- uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix
- with:
- ec-tdf-enabled: true
- key-management: ${{ steps.km-check.outputs.supported }}
- kas-name: km1
- kas-port: 8585
- log-type: json
- root-key: ${{ steps.km-check.outputs.root_key }}
-
- - name: Start additional KM kas (km2)
- id: kas-km2
- if: ${{ steps.multikas.outputs.supported == 'true' }}
- uses: opentdf/platform/test/start-additional-kas@998929e5c66d41f928b90e6af7dbaa0a14302ca6 # watch-sh-fix
- with:
- ec-tdf-enabled: true
- kas-name: km2
- key-management: ${{ steps.km-check.outputs.supported }}
- kas-port: 8686
- log-type: json
- root-key: ${{ steps.km-check.outputs.root_key }}
+ platform-working-dir: ${{ steps.run-platform.outputs.platform-working-dir }}
+ root-key: ${{ steps.test-env.outputs.root-key }}
+ key-management-supported: ${{ steps.test-env.outputs.key-management-supported }}
- name: Run attribute based configuration tests
- if: ${{ steps.multikas.outputs.supported == 'true' }}
+ if: ${{ steps.test-env.outputs.multikas-supported == 'true' }}
run: >-
uv run pytest
-ra
@@ -625,13 +436,14 @@ jobs:
env:
PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
PLATFORM_TAG: ${{ matrix.platform-tag }}
+ OT_ROOT_KEY: ${{ steps.test-env.outputs.root-key }}
PLATFORM_LOG_FILE: "../../${{ steps.run-platform.outputs.platform-log-file }}"
- KAS_ALPHA_LOG_FILE: "../../${{ steps.kas-alpha.outputs.log-file }}"
- KAS_BETA_LOG_FILE: "../../${{ steps.kas-beta.outputs.log-file }}"
- KAS_GAMMA_LOG_FILE: "../../${{ steps.kas-gamma.outputs.log-file }}"
- KAS_DELTA_LOG_FILE: "../../${{ steps.kas-delta.outputs.log-file }}"
- KAS_KM1_LOG_FILE: "../../${{ steps.kas-km1.outputs.log-file }}"
- KAS_KM2_LOG_FILE: "../../${{ steps.kas-km2.outputs.log-file }}"
+ KAS_ALPHA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-alpha-log-file }}
+ KAS_BETA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-beta-log-file }}
+ KAS_GAMMA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-gamma-log-file }}
+ KAS_DELTA_LOG_FILE: ${{ steps.kas-instances.outputs.kas-delta-log-file }}
+ KAS_KM1_LOG_FILE: ${{ steps.kas-instances.outputs.kas-km1-log-file }}
+ KAS_KM2_LOG_FILE: ${{ steps.kas-instances.outputs.kas-km2-log-file }}
- name: Upload artifact
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
diff --git a/otdf-local/src/otdf_local/ci.py b/otdf-local/src/otdf_local/ci.py
new file mode 100644
index 000000000..bab3b3203
--- /dev/null
+++ b/otdf-local/src/otdf_local/ci.py
@@ -0,0 +1,223 @@
+"""CI-specific commands for otdf-local.
+
+These commands adapt the local environment management for GitHub Actions CI,
+where the platform is already started by an external action and we only need
+to start KAS instances as background processes.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from otdf_local.config.ports import Ports
+from otdf_local.config.settings import Settings
+from otdf_local.health.waits import WaitTimeoutError, wait_for_health
+from otdf_local.services import get_kas_manager
+from otdf_local.utils.console import (
+ print_error,
+ print_info,
+ print_success,
+ print_warning,
+)
+from otdf_local.utils.yaml import load_yaml, save_yaml, set_nested
+
+ci_app = typer.Typer(
+ name="ci",
+ help="CI-specific commands for GitHub Actions workflows.",
+ no_args_is_help=True,
+)
+
+
+def _emit_github_output(key: str, value: str) -> None:
+ """Write a key=value pair to $GITHUB_OUTPUT if available, else print to stdout."""
+ github_output = os.environ.get("GITHUB_OUTPUT")
+ if github_output:
+ with open(github_output, "a") as f:
+ f.write(f"{key}={value}\n")
+ else:
+ # Fallback for local testing
+ print(f"{key}={value}", file=sys.stdout)
+
+
+def _prepare_kas_template(
+ settings: Settings, root_key: str | None, ec_tdf_enabled: bool
+) -> None:
+ """Ensure the KAS template config has the right root key and EC TDF settings.
+
+ In CI, the platform config may have a root_key that differs from what
+ we want for additional KAS instances. This updates the platform config
+ in-place so that KASService._generate_config reads the correct root_key.
+ """
+ if root_key:
+ config = load_yaml(settings.platform_config)
+ set_nested(config, "services.kas.root_key", root_key)
+ if ec_tdf_enabled:
+ set_nested(config, "services.kas.preview.ec_tdf_enabled", True)
+ save_yaml(settings.platform_config, config)
+
+
+@ci_app.command("start-kas")
+def start_kas(
+ platform_dir: Annotated[
+ Path,
+ typer.Option(
+ "--platform-dir",
+ help="Path to the platform checkout (must contain opentdf-kas-mode.yaml)",
+ envvar="OTDF_LOCAL_PLATFORM_DIR",
+ ),
+ ],
+ root_key: Annotated[
+ str | None,
+ typer.Option(
+ "--root-key",
+ help="Root key for KAS instances (overrides platform config value)",
+ envvar="OT_ROOT_KEY",
+ ),
+ ] = None,
+ ec_tdf_enabled: Annotated[
+ bool,
+ typer.Option(
+ "--ec-tdf-enabled/--no-ec-tdf",
+ help="Enable EC TDF support",
+ ),
+ ] = True,
+ key_management: Annotated[
+ bool,
+ typer.Option(
+ "--key-management/--no-key-management",
+ help="Enable key management on km1/km2 instances",
+ ),
+ ] = False,
+ log_type: Annotated[
+ str,
+ typer.Option(
+ "--log-type",
+ help="Log format type (json, text)",
+ ),
+ ] = "json",
+ health_timeout: Annotated[
+ int,
+ typer.Option(
+ "--health-timeout",
+ help="Seconds to wait for each KAS instance to become healthy",
+ ),
+ ] = 60,
+ instances: Annotated[
+ str | None,
+ typer.Option(
+ "--instances",
+ help="Comma-separated KAS instance names (default: all)",
+ ),
+ ] = None,
+) -> None:
+ """Start KAS instances for CI and emit GitHub Actions outputs.
+
+ Expects the platform to already be running (started by start-up-with-containers).
+ Starts all 6 KAS instances (alpha, beta, gamma, delta, km1, km2) as background
+ processes, waits for each to pass health checks, and emits log file paths as
+ GitHub Actions step outputs.
+
+ Output keys (written to $GITHUB_OUTPUT):
+ kas-alpha-log-file, kas-beta-log-file, kas-gamma-log-file,
+ kas-delta-log-file, kas-km1-log-file, kas-km2-log-file
+ """
+ platform_dir = platform_dir.resolve()
+ if not platform_dir.is_dir():
+ print_error(f"Platform directory does not exist: {platform_dir}")
+ raise typer.Exit(1)
+
+ # Check for required template files
+ kas_template = platform_dir / "opentdf-kas-mode.yaml"
+ platform_config = platform_dir / "opentdf-dev.yaml"
+ if not kas_template.exists():
+ # Fall back to opentdf.yaml if opentdf-kas-mode.yaml doesn't exist
+ kas_template_alt = platform_dir / "opentdf.yaml"
+ if kas_template_alt.exists():
+ print_info(
+ f"Using {kas_template_alt} as KAS template (opentdf-kas-mode.yaml not found)"
+ )
+ else:
+ print_error(
+ f"Neither opentdf-kas-mode.yaml nor opentdf.yaml found in {platform_dir}"
+ )
+ raise typer.Exit(1)
+
+ if not platform_config.exists():
+ # Try opentdf.yaml as fallback
+ platform_config_alt = platform_dir / "opentdf.yaml"
+ if platform_config_alt.exists():
+ platform_config = platform_config_alt
+
+ # Build settings with CI-specific overrides
+ # We use a fresh xtest_root derived from this package's location
+ settings = Settings(
+ platform_dir=platform_dir,
+ )
+ settings.ensure_directories()
+
+ # Update root key in platform config if provided
+ if root_key:
+ _prepare_kas_template(settings, root_key, ec_tdf_enabled)
+
+ # Determine which instances to start
+ if instances:
+ kas_names = [n.strip() for n in instances.split(",")]
+ for name in kas_names:
+ if name not in Ports.all_kas_names():
+ print_error(f"Unknown KAS instance: {name}")
+ raise typer.Exit(1)
+ else:
+ kas_names = Ports.all_kas_names()
+
+ # Start KAS instances
+ print_info(f"Starting KAS instances: {', '.join(kas_names)}...")
+ kas_manager = get_kas_manager(settings)
+
+ failed = []
+ for name in kas_names:
+ kas = kas_manager.get(name)
+ if kas is None:
+ print_error(f"KAS instance {name} not found in manager")
+ failed.append(name)
+ continue
+ if not kas.start():
+ print_error(f"Failed to start KAS {name}")
+ failed.append(name)
+
+ if failed:
+ print_error(f"Failed to start: {', '.join(failed)}")
+ raise typer.Exit(1)
+
+ # Wait for health
+ print_info("Waiting for KAS health checks...")
+ unhealthy = []
+ for name in kas_names:
+ port = Ports.get_kas_port(name)
+ try:
+ wait_for_health(
+ f"http://localhost:{port}/healthz",
+ timeout=health_timeout,
+ service_name=f"KAS {name}",
+ )
+ except WaitTimeoutError as e:
+ print_warning(str(e))
+ unhealthy.append(name)
+
+ if unhealthy:
+ print_error(f"KAS instances failed health check: {', '.join(unhealthy)}")
+ raise typer.Exit(1)
+
+ print_success(f"All {len(kas_names)} KAS instances are healthy")
+
+ # Emit outputs
+ for name in kas_names:
+ log_path = settings.get_kas_log_path(name)
+ output_key = f"kas-{name}-log-file"
+ _emit_github_output(output_key, str(log_path))
+
+ print_success("CI KAS startup complete")
diff --git a/otdf-local/src/otdf_local/cli.py b/otdf-local/src/otdf_local/cli.py
index d8e3597ff..59fc8e098 100644
--- a/otdf-local/src/otdf_local/cli.py
+++ b/otdf-local/src/otdf_local/cli.py
@@ -11,6 +11,7 @@
from rich.live import Live
from otdf_local import __version__
+from otdf_local.ci import ci_app
from otdf_local.config.ports import Ports
from otdf_local.config.settings import get_settings
from otdf_local.health.waits import WaitTimeoutError, wait_for_health, wait_for_port
@@ -43,6 +44,8 @@
pretty_exceptions_enable=sys.stderr.isatty(),
)
+app.add_typer(ci_app, name="ci")
+
def _show_provision_error(result: ProvisionResult, target: str) -> None:
"""Display provisioning error with stderr details."""
diff --git a/otdf-sdk-mgr/README.md b/otdf-sdk-mgr/README.md
index ee2a2a8b7..4b930eaae 100644
--- a/otdf-sdk-mgr/README.md
+++ b/otdf-sdk-mgr/README.md
@@ -56,7 +56,7 @@ otdf-sdk-mgr java-fixup
## How Release Installs Work
-- **Go**: Writes a `.version` file; `cli.sh`/`otdfctl.sh` use `go run github.com/opentdf/otdfctl@{version}` (no local compilation needed, Go caches the binary)
+- **Go**: Writes a `.version` file containing `module-path@version` (e.g., `github.com/opentdf/otdfctl@v0.24.0`); `cli.sh`/`otdfctl.sh` use `go run @` (no local compilation needed, Go caches the binary). The module path is `github.com/opentdf/platform/otdfctl` for platform-embedded releases or `github.com/opentdf/otdfctl` for standalone releases.
- **JS**: Runs `npm install @opentdf/ctl@{version}` into the dist directory; `cli.sh` uses `npx` from local `node_modules/`
- **Java**: Downloads `cmdline.jar` from GitHub Releases; `cli.sh` uses `java -jar cmdline.jar`
diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py
index 24148bdd7..62580ebc7 100644
--- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py
+++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli.py
@@ -93,3 +93,39 @@ def java_fixup(
from otdf_sdk_mgr.java_fixup import post_checkout_java_fixup
post_checkout_java_fixup(base_dir)
+
+
+@app.command("go-fixup")
+def go_fixup_cmd(
+ platform_dir: Annotated[
+ Path,
+ typer.Option("--platform-dir", help="Path to the platform checkout root"),
+ ],
+ heads: Annotated[
+ Optional[str],
+ typer.Option(
+ "--heads",
+ help="JSON list of head version tags to process (e.g. '[\"main\"]')",
+ ),
+ ] = None,
+ base_dir: Annotated[
+ Optional[Path],
+ typer.Argument(help="Base directory for Go source trees"),
+ ] = None,
+) -> None:
+ """Bridge Go client go.mod to server shared modules for head builds.
+
+ Performs go mod edit -replace + go mod tidy for each head version,
+ pointing platform module imports at the local platform checkout.
+ Only needed for standalone otdfctl checkouts.
+ """
+ import json as json_mod
+
+ from otdf_sdk_mgr.go_fixup import go_fixup
+
+ heads_list = json_mod.loads(heads) if heads else None
+ try:
+ go_fixup(platform_dir, heads=heads_list, base_dir=base_dir)
+ except (FileNotFoundError, subprocess.CalledProcessError) as e:
+ typer.echo(f"Error: {e}", err=True)
+ raise typer.Exit(1) from e
diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py
index e3950d717..e62ae2464 100644
--- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py
+++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py
@@ -74,12 +74,16 @@ def artifact(
dist_name: Annotated[
Optional[str], typer.Option("--dist-name", help="Override dist directory name")
] = None,
+ source: Annotated[
+ Optional[str],
+ typer.Option(help='Source repo for Go CLI (e.g., "platform" for monorepo)'),
+ ] = None,
) -> None:
"""Install a single SDK version (used by CI)."""
from otdf_sdk_mgr.installers import InstallError, cmd_install
try:
- cmd_install(sdk, version, dist_name=dist_name)
+ cmd_install(sdk, version, dist_name=dist_name, source=source)
except InstallError as e:
typer.echo(f"Error: {e}", err=True)
raise typer.Exit(1)
diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py
index 19188b124..2dcf6e321 100644
--- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py
+++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_versions.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
+import os
from typing import Annotated, Any, Optional
import typer
@@ -112,10 +113,20 @@ def resolve_versions(
raise typer.Exit(2)
infix = SDK_TAG_INFIXES.get(sdk)
+ # Allow overriding the Go SDK source via OTDFCTL_SOURCE env var
+ # (standalone otdfctl repo vs platform monorepo)
+ go_source = os.environ.get("OTDFCTL_SOURCE") if sdk == "go" else None
+ if go_source and go_source not in ("standalone", "platform"):
+ typer.echo(
+ f"Warning: unrecognized OTDFCTL_SOURCE={go_source!r}; expected 'platform' or 'standalone'",
+ err=True,
+ )
+ go_source = None
+
results: list[ResolveResult] = []
shas: set[str] = set()
for version in tags:
- v = resolve(sdk, version, infix)
+ v = resolve(sdk, version, infix, go_source=go_source)
if is_resolve_success(v):
env = lookup_additional_options(sdk, v["tag"])
if env:
diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py
index adf6c8b1f..1046a5ef8 100644
--- a/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py
+++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/config.py
@@ -70,7 +70,11 @@ def get_sdk_dirs() -> dict[str, Path]:
"java": "opentdf/java-sdk",
}
-GO_INSTALL_PREFIX = "go run github.com/opentdf/otdfctl"
+GO_INSTALL_PREFIX_STANDALONE = "go run github.com/opentdf/otdfctl"
+GO_INSTALL_PREFIX_PLATFORM = "go run github.com/opentdf/platform/otdfctl"
+
+GO_MODULE_PATH = "github.com/opentdf/otdfctl"
+GO_MODULE_PATH_PLATFORM = "github.com/opentdf/platform/otdfctl"
LTS_VERSIONS: dict[str, str] = {
"go": "0.24.0",
@@ -111,4 +115,46 @@ def get_sdk_dirs() -> dict[str, Path]:
"platform": "service",
}
+# When resolving go versions from the platform repo, use "otdfctl" infix
+# (tags are otdfctl/vX.Y.Z in the platform monorepo)
+SDK_TAG_INFIXES_PLATFORM_GO = "otdfctl"
+
+_VALID_GO_SOURCES = {None, "standalone", "platform"}
+
+
+def _validate_go_source(source: str | None) -> None:
+ """Raise ValueError if source is not a recognised Go source."""
+ if source not in _VALID_GO_SOURCES:
+ raise ValueError(f"Invalid Go source {source!r}; expected one of {_VALID_GO_SOURCES}")
+
+
+def go_git_url(source: str | None = None) -> str:
+ """Return the git URL for Go SDK resolution based on source.
+
+ Args:
+ source: "platform" to use the platform monorepo, None/"standalone" for the
+ standalone otdfctl repo.
+ """
+ _validate_go_source(source)
+ if source == "platform":
+ return SDK_GIT_URLS["platform"]
+ return SDK_GIT_URLS["go"]
+
+
+def go_tag_infix(source: str | None = None) -> str | None:
+ """Return the tag infix for Go SDK resolution based on source."""
+ _validate_go_source(source)
+ if source == "platform":
+ return SDK_TAG_INFIXES_PLATFORM_GO
+ return None
+
+
+def go_module_path(source: str | None = None) -> str:
+ """Return the Go module path based on source."""
+ _validate_go_source(source)
+ if source == "platform":
+ return GO_MODULE_PATH_PLATFORM
+ return GO_MODULE_PATH
+
+
ALL_SDKS = ["go", "js", "java"]
diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py
new file mode 100644
index 000000000..4098361c2
--- /dev/null
+++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/go_fixup.py
@@ -0,0 +1,95 @@
+"""Post-checkout fixups for Go SDK (otdfctl) source trees.
+
+Bridges client go.mod to server shared modules for head builds where
+client and server share unreleased code. Only applies to standalone
+otdfctl checkouts — platform-source builds already have the modules.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from otdf_sdk_mgr.config import get_sdk_dir
+
+# Platform modules that standalone otdfctl imports and that may need
+# a local replace directive when testing against a head platform build.
+PLATFORM_MODULES = [
+ "lib/fixtures",
+ "lib/ocrypto",
+ "protocol/go",
+ "sdk",
+]
+
+
+def go_fixup(
+ platform_dir: Path,
+ heads: list[str] | None = None,
+ base_dir: Path | None = None,
+) -> None:
+ """Replace go.mod references to point at local platform checkout.
+
+ Args:
+ platform_dir: Absolute path to the platform checkout root
+ (containing lib/, protocol/, sdk/).
+ heads: JSON-decoded list of head version tags to process.
+ If None, all subdirectories under *base_dir* are processed.
+ base_dir: Directory containing per-version otdfctl source trees
+ (e.g. ``xtest/sdk/go/src``). Defaults to ``get_sdk_dir() / "go" / "src"``.
+ """
+ if base_dir is None:
+ base_dir = get_sdk_dir() / "go" / "src"
+
+ if not base_dir.exists():
+ print(f"Base directory {base_dir} does not exist, nothing to fix.")
+ return
+
+ platform_dir = platform_dir.resolve()
+ if not platform_dir.is_dir():
+ raise FileNotFoundError(f"Platform directory does not exist: {platform_dir}")
+
+ dirs_to_process: list[Path] = []
+ if heads:
+ for tag in heads:
+ d = base_dir / tag
+ if d.is_dir():
+ dirs_to_process.append(d)
+ else:
+ print(f"Warning: head directory {d} does not exist, skipping.")
+ else:
+ for d in sorted(base_dir.iterdir()):
+ if d.is_dir() and not d.name.endswith(".git"):
+ dirs_to_process.append(d)
+
+ if not dirs_to_process:
+ print("No directories to process.")
+ return
+
+ for src_dir in dirs_to_process:
+ if not (src_dir / "go.mod").exists():
+ print(f"No go.mod in {src_dir}, skipping.")
+ continue
+
+ print(f"Applying go.mod replacements in {src_dir}...")
+ for module in PLATFORM_MODULES:
+ local_path = platform_dir / module
+ if not local_path.is_dir():
+ print(f" Warning: {local_path} does not exist, skipping {module}")
+ continue
+ subprocess.run(
+ [
+ "go",
+ "mod",
+ "edit",
+ "-replace",
+ f"github.com/opentdf/platform/{module}={local_path}",
+ ],
+ cwd=src_dir,
+ check=True,
+ )
+ print(f" Replaced github.com/opentdf/platform/{module} -> {local_path}")
+
+ print(f"Running go mod tidy in {src_dir}...")
+ subprocess.run(["go", "mod", "tidy"], cwd=src_dir, check=True)
+
+ print("Go fixup complete.")
diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py
index e7c22ae09..0822a063e 100644
--- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py
+++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py
@@ -11,9 +11,11 @@
from pathlib import Path
from otdf_sdk_mgr.config import (
+ GO_MODULE_PATH_PLATFORM,
LTS_VERSIONS,
get_sdk_dir,
get_sdk_dirs,
+ go_module_path,
)
from otdf_sdk_mgr.checkout import checkout_sdk_branch
from otdf_sdk_mgr.registry import list_go_versions, list_java_github_releases, list_js_versions
@@ -24,33 +26,48 @@ class InstallError(Exception):
"""Raised when SDK installation fails."""
-def install_go_release(version: str, dist_dir: Path) -> None:
+def install_go_release(version: str, dist_dir: Path, source: str | None = None) -> None:
"""Install a Go CLI release by writing a .version file.
The cli.sh and otdfctl.sh wrappers read .version and use
- `go run github.com/opentdf/otdfctl@{version}` instead of a local binary.
+ `go run @{version}` instead of a local binary.
+ The .version file contains `module-path@version`
+ (e.g., `github.com/opentdf/otdfctl@v0.24.0`).
+
+ Args:
+ version: Version string (e.g., "v0.24.0" or "otdfctl/v0.24.0").
+ dist_dir: Target distribution directory.
+ source: "platform" to use the platform monorepo module path,
+ None or "standalone" for standalone.
"""
go_dir = get_sdk_dir() / "go"
dist_dir.mkdir(parents=True, exist_ok=True)
+ # Strip tag infix (e.g., "otdfctl/v0.24.0" → "v0.24.0")
+ if "/" in version:
+ version = version.rsplit("/", 1)[-1]
tag = normalize_version(version)
- (dist_dir / ".version").write_text(f"{tag}\n")
+ module = go_module_path(source)
+ (dist_dir / ".version").write_text(f"{module}@{tag}\n")
shutil.copy(go_dir / "cli.sh", dist_dir / "cli.sh")
shutil.copy(go_dir / "otdfctl.sh", dist_dir / "otdfctl.sh")
shutil.copy(go_dir / "opentdfctl.yaml", dist_dir / "opentdfctl.yaml")
- print(f" Pre-warming Go cache for otdfctl@{tag}...")
+ print(f" Pre-warming Go cache for {module}@{tag}...")
result = subprocess.run(
- ["go", "install", f"github.com/opentdf/otdfctl@{tag}"],
+ ["go", "install", f"{module}@{tag}"],
capture_output=True,
text=True,
)
if result.returncode != 0:
- print(
- f" Warning: go install pre-warm failed (will retry at runtime): {result.stderr.strip()}"
- )
+ msg = f"go install pre-warm failed: {result.stderr.strip()}"
+ if module == GO_MODULE_PATH_PLATFORM:
+ raise InstallError(
+ f"{msg}\nThe platform module path {module}@{tag} may not be published yet."
+ )
+ print(f" Warning: {msg} (will retry at runtime)")
print(f" Go release {tag} installed to {dist_dir}")
-def install_js_release(version: str, dist_dir: Path) -> None:
+def install_js_release(version: str, dist_dir: Path, **_kwargs: object) -> None:
"""Install a JS CLI release from npm registry."""
js_dir = get_sdk_dir() / "js"
dist_dir.mkdir(parents=True, exist_ok=True)
@@ -65,7 +82,7 @@ def install_js_release(version: str, dist_dir: Path) -> None:
print(f" JS release {v} installed to {dist_dir}")
-def install_java_release(version: str, dist_dir: Path) -> None:
+def install_java_release(version: str, dist_dir: Path, **_kwargs: object) -> None:
"""Install a Java CLI release by downloading cmdline.jar from GitHub Releases.
Raises InstallError if the artifact is not available or download fails,
@@ -133,13 +150,15 @@ def install_java_release(version: str, dist_dir: Path) -> None:
}
-def install_release(sdk: str, version: str, dist_name: str | None = None) -> Path:
+def install_release(sdk: str, version: str, dist_name: str | None = None, **kwargs: object) -> Path:
"""Install a released version of an SDK CLI.
Args:
sdk: One of "go", "js", "java"
version: Version string (e.g., "v0.24.0" or "0.24.0")
dist_name: Override the dist directory name (defaults to normalized version)
+ **kwargs: Extra arguments forwarded to the SDK installer
+ (e.g., source="platform" for Go).
Returns:
Path to the created dist directory
@@ -157,7 +176,7 @@ def install_release(sdk: str, version: str, dist_name: str | None = None) -> Pat
print(f" Dist directory already exists: {dist_dir} (skipping)")
return dist_dir
- INSTALLERS[sdk](version, dist_dir)
+ INSTALLERS[sdk](version, dist_dir, **kwargs)
return dist_dir
@@ -224,7 +243,9 @@ def cmd_release(specs: list[str]) -> None:
install_release(sdk, version)
-def cmd_install(sdk: str, version: str, dist_name: str | None = None) -> None:
+def cmd_install(
+ sdk: str, version: str, dist_name: str | None = None, source: str | None = None
+) -> None:
"""Install a single SDK version (used by CI action)."""
print(f"Installing {sdk} {version}...")
- install_release(sdk, version, dist_name=dist_name)
+ install_release(sdk, version, dist_name=dist_name, source=source)
diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py
index 8f8dd34e5..fcd4f78c8 100644
--- a/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py
+++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/registry.py
@@ -12,11 +12,13 @@
from typing import Any
from otdf_sdk_mgr.config import (
- GO_INSTALL_PREFIX,
+ GO_INSTALL_PREFIX_PLATFORM,
+ GO_INSTALL_PREFIX_STANDALONE,
SDK_GITHUB_REPOS,
SDK_GIT_URLS,
SDK_MAVEN_COORDS,
SDK_NPM_PACKAGES,
+ SDK_TAG_INFIXES_PLATFORM_GO,
)
from otdf_sdk_mgr.semver import is_stable, parse_semver, semver_sort_key
@@ -68,12 +70,15 @@ def fetch_text(url: str) -> str:
def list_go_versions() -> list[dict[str, Any]]:
- """List Go SDK versions from git tags."""
+ """List Go SDK versions from git tags in both standalone and platform repos."""
+ import git.exc
from git import Git
repo = Git()
+ seen: dict[str, dict[str, Any]] = {}
+
+ # Standalone repo (opentdf/otdfctl): tags like v0.24.0
raw = repo.ls_remote(SDK_GIT_URLS["go"], tags=True)
- results = []
for line in raw.strip().split("\n"):
if not line:
continue
@@ -83,16 +88,52 @@ def list_go_versions() -> list[dict[str, Any]]:
tag = ref.removeprefix("refs/tags/")
if not parse_semver(tag):
continue
- version = tag
- results.append(
- {
+ seen[tag] = {
+ "sdk": "go",
+ "version": tag,
+ "source": "git-tag",
+ "install_method": f"{GO_INSTALL_PREFIX_STANDALONE}@{tag}",
+ "stable": is_stable(tag),
+ }
+
+ # Platform repo (opentdf/platform): tags like otdfctl/v0.X.Y
+ infix = SDK_TAG_INFIXES_PLATFORM_GO
+ try:
+ raw = repo.ls_remote(SDK_GIT_URLS["platform"], tags=True)
+ for line in raw.strip().split("\n"):
+ if not line:
+ continue
+ _, ref = line.split("\t", 1)
+ if ref.endswith("^{}"):
+ continue
+ tag = ref.removeprefix("refs/tags/")
+ if not tag.startswith(f"{infix}/"):
+ continue
+ version = tag.removeprefix(f"{infix}/")
+ if not parse_semver(version):
+ continue
+ # Platform entries take precedence (canonical location post-migration);
+ # if the same version exists in both repos, the platform entry
+ # silently overwrites the standalone one.
+ if version in seen:
+ print(
+ f"Note: version {version} found in both standalone and platform repos; using platform source.",
+ file=sys.stderr,
+ )
+ seen[version] = {
"sdk": "go",
"version": version,
- "source": "git-tag",
- "install_method": f"{GO_INSTALL_PREFIX}@{version}",
+ "source": "platform-git-tag",
+ "install_method": f"{GO_INSTALL_PREFIX_PLATFORM}@{tag}",
"stable": is_stable(version),
}
+ except git.exc.GitCommandError as e:
+ print(
+ f"::warning::Failed to query platform repo for go tags: {e}",
+ file=sys.stderr,
)
+
+ results = list(seen.values())
results.sort(key=lambda r: semver_sort_key(r["version"]))
return results
diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py
index 6e4cd7ca1..e3f264d74 100644
--- a/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py
+++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/resolve.py
@@ -12,6 +12,8 @@
LTS_VERSIONS,
SDK_GIT_URLS,
SDK_NPM_PACKAGES,
+ go_git_url,
+ go_tag_infix,
)
@@ -23,6 +25,7 @@ class ResolveSuccess(TypedDict):
pr: NotRequired[str]
release: NotRequired[str]
sha: str
+ source: NotRequired[str]
tag: str
@@ -111,78 +114,119 @@ def lookup_additional_options(sdk: str, version: str) -> str | None:
return None
-def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult:
- """Resolve a version spec to a concrete SHA and tag."""
+def resolve(
+ sdk: str,
+ version: str,
+ infix: str | None,
+ go_source: str | None = None,
+) -> ResolveResult:
+ """Resolve a version spec to a concrete SHA and tag.
+
+ Args:
+ sdk: SDK identifier (go, js, java, platform).
+ version: Version spec (main, SHA, tag, latest, lts, etc.).
+ infix: Tag infix for monorepo tag resolution (e.g. "sdk" for JS).
+ go_source: For sdk=="go", override the git URL and infix.
+ "platform" resolves against the platform monorepo (otdfctl/ prefix tags).
+ None or "standalone" uses the standalone otdfctl repo (default).
+ """
+ _go_platform = sdk == "go" and go_source == "platform"
+
+ def _annotate(result: ResolveResult) -> ResolveResult:
+ """Add source field to successful results when resolving go from platform."""
+ if _go_platform and is_resolve_success(result):
+ result["source"] = "platform"
+ return result
+
try:
- sdk_url = SDK_GIT_URLS[sdk]
+ if _go_platform:
+ sdk_url = go_git_url("platform")
+ infix = go_tag_infix("platform")
+ else:
+ sdk_url = SDK_GIT_URLS[sdk]
repo = Git()
if version == "main" or version == "refs/heads/main":
all_heads = [r.split("\t") for r in repo.ls_remote(sdk_url, heads=True).split("\n")]
- sha, _ = [tag for tag in all_heads if "refs/heads/main" in tag][0]
- return {
- "sdk": sdk,
- "alias": version,
- "head": True,
- "sha": sha,
- "tag": "main",
- }
+ try:
+ sha, _ = next(tag for tag in all_heads if "refs/heads/main" in tag)
+ except StopIteration:
+ return {"sdk": sdk, "alias": version, "err": f"main branch not found in {sdk_url}"}
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": version,
+ "head": True,
+ "sha": sha,
+ "tag": "main",
+ }
+ )
if re.match(SHA_REGEX, version):
ls_remote = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")]
matching_tags = [(sha, tag) for (sha, tag) in ls_remote if sha.startswith(version)]
if not matching_tags:
- return {
- "sdk": sdk,
- "alias": version[:7],
- "sha": version,
- "tag": version,
- }
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": version[:7],
+ "sha": version,
+ "tag": version,
+ }
+ )
if len(matching_tags) > 1:
for sha, tag in matching_tags:
if tag.startswith("refs/pull/"):
pr_number = tag.split("/")[2]
- return {
- "sdk": sdk,
- "alias": version,
- "head": True,
- "sha": sha,
- "tag": f"pull-{pr_number}",
- }
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": version,
+ "head": True,
+ "sha": sha,
+ "tag": f"pull-{pr_number}",
+ }
+ )
for sha, tag in matching_tags:
mq_match = re.match(MERGE_QUEUE_REGEX, tag)
if mq_match:
to_branch = mq_match.group("branch")
pr_number = mq_match.group("pr_number")
if to_branch and pr_number:
- return {
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": version,
+ "head": True,
+ "pr": pr_number,
+ "sha": sha,
+ "tag": f"mq-{to_branch}-{pr_number}",
+ }
+ )
+ suffix = tag.split("refs/heads/gh-readonly-queue/")[-1]
+ flattag = "mq--" + suffix.replace("/", "--")
+ return _annotate(
+ {
"sdk": sdk,
"alias": version,
"head": True,
- "pr": pr_number,
"sha": sha,
- "tag": f"mq-{to_branch}-{pr_number}",
+ "tag": flattag,
}
- suffix = tag.split("refs/heads/gh-readonly-queue/")[-1]
- flattag = "mq--" + suffix.replace("/", "--")
- return {
- "sdk": sdk,
- "alias": version,
- "head": True,
- "sha": sha,
- "tag": flattag,
- }
+ )
head = False
if tag.startswith("refs/heads/"):
head = True
tag = tag.split("refs/heads/")[-1]
flattag = tag.replace("/", "--")
- return {
- "sdk": sdk,
- "alias": version,
- "head": head,
- "sha": sha,
- "tag": flattag,
- }
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": version,
+ "head": head,
+ "sha": sha,
+ "tag": flattag,
+ }
+ )
return {
"sdk": sdk,
@@ -197,12 +241,14 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult:
tag = tag.split("refs/tags/")[-1]
if infix:
tag = tag.split(f"{infix}/")[-1]
- return {
- "sdk": sdk,
- "alias": version,
- "sha": sha,
- "tag": tag,
- }
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": version,
+ "sha": sha,
+ "tag": tag,
+ }
+ )
if version.startswith("refs/pull/"):
merge_heads = [
@@ -216,14 +262,16 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult:
"err": f"pull request {pr_number} not found in {sdk_url}",
}
sha, _ = merge_heads[0]
- return {
- "sdk": sdk,
- "alias": version,
- "head": True,
- "pr": pr_number,
- "sha": sha,
- "tag": f"pull-{pr_number}",
- }
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": version,
+ "head": True,
+ "pr": pr_number,
+ "sha": sha,
+ "tag": f"pull-{pr_number}",
+ }
+ )
remote_tags = [r.split("\t") for r in repo.ls_remote(sdk_url).split("\n")]
all_listed_tags = [
@@ -238,13 +286,15 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult:
if version in all_listed_branches:
sha = all_listed_branches[version]
- return {
- "sdk": sdk,
- "alias": version,
- "head": True,
- "sha": sha,
- "tag": version,
- }
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": version,
+ "head": True,
+ "sha": sha,
+ "tag": version,
+ }
+ )
if infix and version.startswith(f"{infix}/"):
version = version.split(f"{infix}/")[-1]
@@ -288,13 +338,15 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult:
if not matching_tags:
# No versions with CLI found, fall back to building latest from source
sha, tag = stable_tags[-1]
- return {
- "sdk": sdk,
- "alias": alias,
- "head": True, # Mark as head to trigger source checkout
- "sha": sha,
- "tag": tag,
- }
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": alias,
+ "head": True, # Mark as head to trigger source checkout
+ "sha": sha,
+ "tag": tag,
+ }
+ )
else:
matching_tags = stable_tags[-1:]
else:
@@ -319,13 +371,15 @@ def resolve(sdk: str, version: str, infix: str | None) -> ResolveResult:
release = tag
if infix:
release = f"{infix}/{release}"
- return {
- "sdk": sdk,
- "alias": alias,
- "release": release,
- "sha": sha,
- "tag": tag,
- }
+ return _annotate(
+ {
+ "sdk": sdk,
+ "alias": alias,
+ "release": release,
+ "sha": sha,
+ "tag": tag,
+ }
+ )
except Exception as e:
return {
"sdk": sdk,
diff --git a/xtest/sdk/go/cli.sh b/xtest/sdk/go/cli.sh
index 172aa5b50..f97b20c1a 100755
--- a/xtest/sdk/go/cli.sh
+++ b/xtest/sdk/go/cli.sh
@@ -23,8 +23,14 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
cmd=("$SCRIPT_DIR"/otdfctl)
if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then
if [ -f "$SCRIPT_DIR/.version" ]; then
- OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version")
- cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}")
+ VERSION_SPEC=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version")
+ if [[ "$VERSION_SPEC" == *@* ]]; then
+ # New format: module-path@version
+ cmd=(go run "$VERSION_SPEC")
+ else
+ # Legacy format: bare version tag, default to standalone module
+ cmd=(go run "github.com/opentdf/otdfctl@${VERSION_SPEC}")
+ fi
else
cmd=(go run "github.com/opentdf/otdfctl@latest")
fi
diff --git a/xtest/sdk/go/otdfctl.sh b/xtest/sdk/go/otdfctl.sh
index 17fbb0c84..9ba55f054 100755
--- a/xtest/sdk/go/otdfctl.sh
+++ b/xtest/sdk/go/otdfctl.sh
@@ -18,8 +18,14 @@ source "$XTEST_DIR/test.env"
cmd=("$SCRIPT_DIR"/otdfctl)
if [ ! -f "$SCRIPT_DIR"/otdfctl ]; then
if [ -f "$SCRIPT_DIR/.version" ]; then
- OTDFCTL_VERSION=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version")
- cmd=(go run "github.com/opentdf/otdfctl@${OTDFCTL_VERSION}")
+ VERSION_SPEC=$(tr -d '[:space:]' <"$SCRIPT_DIR/.version")
+ if [[ "$VERSION_SPEC" == *@* ]]; then
+ # New format: module-path@version
+ cmd=(go run "$VERSION_SPEC")
+ else
+ # Legacy format: bare version tag, default to standalone module
+ cmd=(go run "github.com/opentdf/otdfctl@${VERSION_SPEC}")
+ fi
else
cmd=(go run "github.com/opentdf/otdfctl@latest")
fi
diff --git a/xtest/setup-cli-tool/action.yaml b/xtest/setup-cli-tool/action.yaml
index 9e110ef4f..5bf6eff62 100644
--- a/xtest/setup-cli-tool/action.yaml
+++ b/xtest/setup-cli-tool/action.yaml
@@ -2,12 +2,23 @@ name: configure-sdks
description: Check out and build one or more SDK and its CLI tool for use by xtest
inputs:
path:
- description: The path to checkout the the SDK source code to; concatenated with branch or tag name.
+ description: The path to check out the SDK source code to; concatenated with branch or tag name.
sdk:
description: The SDK to configure; one of go, java, js
version-info:
description: JSON-encoded output of otdf-sdk-mgr versions resolve
required: true
+ platform-otdfctl-dir:
+ description: >-
+ Absolute path to platform's otdfctl/ directory. When set and sdk is "go",
+ head versions whose SHA matches platform-otdfctl-sha are symlinked from
+ here instead of checked out separately. Used in both explicit platform
+ mode (source: "platform" in resolved version) and auto-detect mode.
+ platform-otdfctl-sha:
+ description: >-
+ SHA of the commit that platform-otdfctl-dir was checked out at.
+ Used to decide which Go head version (if any) can reuse the existing
+ platform checkout vs needing a fresh one.
outputs:
version-a:
description: "Object containing tag, sha, and name of a version checked out"
@@ -28,24 +39,27 @@ outputs:
runs:
using: composite
steps:
- - name: identify repo url
+ - name: identify repo urls
shell: bash
run: |
- case "${{ inputs.sdk }}" in
+ case "$INPUT_SDK" in
"go")
- echo "sdk_repo=opentdf/otdfctl" >> $GITHUB_ENV
+ echo "STANDALONE_REPO=opentdf/otdfctl" >> "$GITHUB_ENV"
;;
"java")
- echo "sdk_repo=opentdf/java-sdk" >> $GITHUB_ENV
+ echo "STANDALONE_REPO=opentdf/java-sdk" >> "$GITHUB_ENV"
;;
"js")
- echo "sdk_repo=opentdf/web-sdk" >> $GITHUB_ENV
+ echo "STANDALONE_REPO=opentdf/web-sdk" >> "$GITHUB_ENV"
;;
*)
- echo "Invalid SDK specified: ${{ inputs.sdk }}" >> $GITHUB_STEP_SUMMARY
+ echo "Invalid SDK specified: $INPUT_SDK" >> "$GITHUB_STEP_SUMMARY"
exit 1
;;
esac
+ echo "PLATFORM_REPO=opentdf/platform" >> "$GITHUB_ENV"
+ env:
+ INPUT_SDK: ${{ inputs.sdk }}
- name: resolve versions
id: resolve
@@ -88,9 +102,12 @@ runs:
echo "Installing ${{ inputs.sdk }} $tag from registry (release: $release)"
# Sanitize tag for use as an env var name (replace non-alphanumeric/underscore with _)
tag_sanitized="${tag//[^a-zA-Z0-9_]/_}"
+ source=$(echo "$row" | jq -r '.source // empty')
+ source_args=()
+ [[ -n "$source" ]] && source_args=(--source "$source")
if ! uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr install artifact \
--sdk "${{ inputs.sdk }}" --version "$release" \
- --dist-name "$tag"; then
+ --dist-name "$tag" "${source_args[@]}"; then
echo " Warning: Artifact installation failed for ${{ inputs.sdk }} $tag"
echo " Will fall back to building from source"
echo "BUILD_FROM_SOURCE_${tag_sanitized}=true" >> "$GITHUB_ENV"
@@ -104,9 +121,10 @@ runs:
id: check-source
shell: bash
run: |
- # Determine which version slots need source checkout.
- # A slot needs checkout if it is a head version OR if artifact install failed
- # (BUILD_FROM_SOURCE_ was set in the previous step).
+ # Determine which version slots need source checkout and from which repo.
+ # A slot needs checkout if it is a head version OR if artifact install failed.
+ # Platform-source versions may reuse the existing platform-otdfctl-dir
+ # (when their SHA matches) or need a fresh opentdf/platform checkout.
for slot in a b c d; do
case "$slot" in
a) row=$(echo "${version_info}" | jq -rc '.[0] // empty') ;;
@@ -115,65 +133,200 @@ runs:
d) row=$(echo "${version_info}" | jq -rc '.[3] // empty') ;;
esac
if [[ -z "$row" ]]; then
- echo "needs-source-${slot}=false" >> "$GITHUB_OUTPUT"
+ echo "needs-checkout-${slot}=false" >> "$GITHUB_OUTPUT"
+ echo "is-platform-${slot}=false" >> "$GITHUB_OUTPUT"
+ echo "use-existing-platform-dir-${slot}=false" >> "$GITHUB_OUTPUT"
+ echo "checkout-repo-${slot}=" >> "$GITHUB_OUTPUT"
+ echo "checkout-path-${slot}=" >> "$GITHUB_OUTPUT"
continue
fi
+
tag=$(echo "$row" | jq -r '.tag')
head=$(echo "$row" | jq -r '.head // false')
+ sha=$(echo "$row" | jq -r '.sha')
+ source=$(echo "$row" | jq -r '.source // empty')
tag_sanitized="${tag//[^a-zA-Z0-9_]/_}"
build_from_source_var="BUILD_FROM_SOURCE_${tag_sanitized}"
+ needs_source=false
if [[ "$head" == "true" || "${!build_from_source_var}" == "true" ]]; then
- echo "needs-source-${slot}=true" >> "$GITHUB_OUTPUT"
- else
- echo "needs-source-${slot}=false" >> "$GITHUB_OUTPUT"
+ needs_source=true
+ fi
+
+ is_platform=false
+ use_existing=false
+ checkout_repo="$STANDALONE_REPO"
+ checkout_path="${INPUT_PATH}/${INPUT_SDK}/src/${tag}"
+
+ if [[ "$source" == "platform" ]]; then
+ # Explicit platform mode: resolver tagged this version as from
+ # opentdf/platform. Use per-version SHA to decide checkout strategy.
+ is_platform=true
+ if [[ "$needs_source" == "true" && -n "$PLATFORM_OTDFCTL_DIR" \
+ && -n "$PLATFORM_OTDFCTL_SHA" && "$sha" == "$PLATFORM_OTDFCTL_SHA" ]]; then
+ # SHA matches existing platform checkout — reuse via symlink
+ use_existing=true
+ needs_source=false
+ elif [[ "$needs_source" == "true" ]]; then
+ # Different SHA — need a fresh platform checkout
+ checkout_repo="$PLATFORM_REPO"
+ checkout_path="${INPUT_PATH}/${INPUT_SDK}/platform-src/${tag}"
+ fi
+ elif [[ "$INPUT_SDK" == "go" && -n "$PLATFORM_OTDFCTL_DIR" && "$needs_source" == "true" ]]; then
+ # Auto-detect fallback: resolver used standalone repo but the
+ # test job detected otdfctl in the platform checkout.
+ # NOTE: SHA comparison across repos is not meaningful (the standalone
+ # repo and platform repo have different commit histories), so we
+ # cannot safely reuse the platform checkout here. Fall through to
+ # a standalone checkout. To use the platform source, set
+ # otdfctl-source=platform explicitly.
+ echo "::notice::Go version ${tag} resolved from standalone repo; platform checkout available but cannot auto-reuse (different repo). Set otdfctl-source=platform to use the platform source."
fi
+
+ echo "needs-checkout-${slot}=${needs_source}" >> "$GITHUB_OUTPUT"
+ echo "is-platform-${slot}=${is_platform}" >> "$GITHUB_OUTPUT"
+ echo "use-existing-platform-dir-${slot}=${use_existing}" >> "$GITHUB_OUTPUT"
+ echo "checkout-repo-${slot}=${checkout_repo}" >> "$GITHUB_OUTPUT"
+ echo "checkout-path-${slot}=${checkout_path}" >> "$GITHUB_OUTPUT"
done
env:
version_info: ${{ inputs.version-info }}
+ INPUT_PATH: ${{ inputs.path }}
+ INPUT_SDK: ${{ inputs.sdk }}
+ PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }}
+ PLATFORM_OTDFCTL_SHA: ${{ inputs.platform-otdfctl-sha }}
+
+ - name: symlink existing platform checkout
+ shell: bash
+ run: |
+ # For versions that can reuse the already-checked-out platform dir,
+ # symlink platform-otdfctl-dir into sdk/go/src/{tag}.
+ for slot in a b c d; do
+ case "$slot" in
+ a) version_json="$VERSION_A" ; use_existing="$USE_EXISTING_A" ;;
+ b) version_json="$VERSION_B" ; use_existing="$USE_EXISTING_B" ;;
+ c) version_json="$VERSION_C" ; use_existing="$USE_EXISTING_C" ;;
+ d) version_json="$VERSION_D" ; use_existing="$USE_EXISTING_D" ;;
+ esac
+ if [[ -z "$version_json" || "$use_existing" != "true" ]]; then
+ continue
+ fi
+ tag=$(echo "$version_json" | jq -r '.tag')
+ src_dir="${INPUT_PATH}/${INPUT_SDK}/src/${tag}"
+ echo "Symlinking existing platform otdfctl to ${src_dir}"
+ mkdir -p "$(dirname "$src_dir")"
+ ln -sfn "$PLATFORM_OTDFCTL_DIR" "$src_dir"
+ if [ ! -e "$src_dir" ]; then
+ echo "::error::Symlink target does not exist: $PLATFORM_OTDFCTL_DIR"
+ exit 1
+ fi
+ done
+ env:
+ PLATFORM_OTDFCTL_DIR: ${{ inputs.platform-otdfctl-dir }}
+ INPUT_PATH: ${{ inputs.path }}
+ INPUT_SDK: ${{ inputs.sdk }}
+ VERSION_A: ${{ steps.resolve.outputs.version-a }}
+ VERSION_B: ${{ steps.resolve.outputs.version-b }}
+ VERSION_C: ${{ steps.resolve.outputs.version-c }}
+ VERSION_D: ${{ steps.resolve.outputs.version-d }}
+ USE_EXISTING_A: ${{ steps.check-source.outputs.use-existing-platform-dir-a }}
+ USE_EXISTING_B: ${{ steps.check-source.outputs.use-existing-platform-dir-b }}
+ USE_EXISTING_C: ${{ steps.check-source.outputs.use-existing-platform-dir-c }}
+ USE_EXISTING_D: ${{ steps.check-source.outputs.use-existing-platform-dir-d }}
- name: checkout version a
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
if: >-
steps.resolve.outputs.version-a != ''
- && steps.check-source.outputs.needs-source-a == 'true'
+ && steps.check-source.outputs.needs-checkout-a == 'true'
with:
- path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-a).tag }}
+ path: ${{ steps.check-source.outputs.checkout-path-a }}
persist-credentials: false
- ref: ${{ fromJson(steps.resolve.outputs.version-a).sha }}
- repository: ${{ env.sdk_repo }}
+ ref: ${{ fromJson(steps.resolve.outputs.version-a || '{"sha":""}').sha }}
+ repository: ${{ steps.check-source.outputs.checkout-repo-a }}
- name: checkout version b
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
if: >-
steps.resolve.outputs.version-b != ''
- && steps.check-source.outputs.needs-source-b == 'true'
+ && steps.check-source.outputs.needs-checkout-b == 'true'
with:
- path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-b).tag }}
+ path: ${{ steps.check-source.outputs.checkout-path-b }}
persist-credentials: false
- ref: ${{ fromJson(steps.resolve.outputs.version-b).sha }}
- repository: ${{ env.sdk_repo }}
+ ref: ${{ fromJson(steps.resolve.outputs.version-b || '{"sha":""}').sha }}
+ repository: ${{ steps.check-source.outputs.checkout-repo-b }}
- name: checkout version c
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
if: >-
steps.resolve.outputs.version-c != ''
- && steps.check-source.outputs.needs-source-c == 'true'
+ && steps.check-source.outputs.needs-checkout-c == 'true'
with:
- path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-c).tag }}
+ path: ${{ steps.check-source.outputs.checkout-path-c }}
persist-credentials: false
- ref: ${{ fromJson(steps.resolve.outputs.version-c).sha }}
- repository: ${{ env.sdk_repo }}
+ ref: ${{ fromJson(steps.resolve.outputs.version-c || '{"sha":""}').sha }}
+ repository: ${{ steps.check-source.outputs.checkout-repo-c }}
- name: checkout version d
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
if: >-
steps.resolve.outputs.version-d != ''
- && steps.check-source.outputs.needs-source-d == 'true'
+ && steps.check-source.outputs.needs-checkout-d == 'true'
with:
- path: ${{ inputs.path }}/${{ inputs.sdk }}/src/${{ fromJson(steps.resolve.outputs.version-d).tag }}
+ path: ${{ steps.check-source.outputs.checkout-path-d }}
persist-credentials: false
- ref: ${{ fromJson(steps.resolve.outputs.version-d).sha }}
- repository: ${{ env.sdk_repo }}
+ ref: ${{ fromJson(steps.resolve.outputs.version-d || '{"sha":""}').sha }}
+ repository: ${{ steps.check-source.outputs.checkout-repo-d }}
+
+ - name: symlink freshly checked-out platform sources
+ shell: bash
+ run: |
+ # For platform-source versions that were checked out (not reusing the
+ # existing dir), symlink {platform-src}/{tag}/otdfctl → src/{tag} so
+ # the Makefile discovers them.
+ for slot in a b c d; do
+ case "$slot" in
+ a) is_platform="$IS_PLATFORM_A" ; needs_checkout="$NEEDS_CHECKOUT_A"
+ checkout_path="$CHECKOUT_PATH_A" ; version_json="$VERSION_A" ;;
+ b) is_platform="$IS_PLATFORM_B" ; needs_checkout="$NEEDS_CHECKOUT_B"
+ checkout_path="$CHECKOUT_PATH_B" ; version_json="$VERSION_B" ;;
+ c) is_platform="$IS_PLATFORM_C" ; needs_checkout="$NEEDS_CHECKOUT_C"
+ checkout_path="$CHECKOUT_PATH_C" ; version_json="$VERSION_C" ;;
+ d) is_platform="$IS_PLATFORM_D" ; needs_checkout="$NEEDS_CHECKOUT_D"
+ checkout_path="$CHECKOUT_PATH_D" ; version_json="$VERSION_D" ;;
+ esac
+ if [[ "$is_platform" != "true" || "$needs_checkout" != "true" || -z "$version_json" ]]; then
+ continue
+ fi
+ tag=$(echo "$version_json" | jq -r '.tag')
+ src_dir="${INPUT_PATH}/${INPUT_SDK}/src/${tag}"
+ otdfctl_dir="${checkout_path}/otdfctl"
+ echo "Symlinking freshly checked-out platform otdfctl ${otdfctl_dir} → ${src_dir}"
+ mkdir -p "$(dirname "$src_dir")"
+ ln -sfn "$otdfctl_dir" "$src_dir"
+ if [ ! -e "$src_dir" ]; then
+ echo "::error::Symlink target does not exist: ${otdfctl_dir} (does the platform repo contain an otdfctl/ directory?)"
+ exit 1
+ fi
+ done
+ env:
+ INPUT_PATH: ${{ inputs.path }}
+ INPUT_SDK: ${{ inputs.sdk }}
+ VERSION_A: ${{ steps.resolve.outputs.version-a }}
+ VERSION_B: ${{ steps.resolve.outputs.version-b }}
+ VERSION_C: ${{ steps.resolve.outputs.version-c }}
+ VERSION_D: ${{ steps.resolve.outputs.version-d }}
+ IS_PLATFORM_A: ${{ steps.check-source.outputs.is-platform-a }}
+ IS_PLATFORM_B: ${{ steps.check-source.outputs.is-platform-b }}
+ IS_PLATFORM_C: ${{ steps.check-source.outputs.is-platform-c }}
+ IS_PLATFORM_D: ${{ steps.check-source.outputs.is-platform-d }}
+ NEEDS_CHECKOUT_A: ${{ steps.check-source.outputs.needs-checkout-a }}
+ NEEDS_CHECKOUT_B: ${{ steps.check-source.outputs.needs-checkout-b }}
+ NEEDS_CHECKOUT_C: ${{ steps.check-source.outputs.needs-checkout-c }}
+ NEEDS_CHECKOUT_D: ${{ steps.check-source.outputs.needs-checkout-d }}
+ CHECKOUT_PATH_A: ${{ steps.check-source.outputs.checkout-path-a }}
+ CHECKOUT_PATH_B: ${{ steps.check-source.outputs.checkout-path-b }}
+ CHECKOUT_PATH_C: ${{ steps.check-source.outputs.checkout-path-c }}
+ CHECKOUT_PATH_D: ${{ steps.check-source.outputs.checkout-path-d }}
- name: post checkout cleanups
if: inputs.sdk == 'java'
diff --git a/xtest/setup-kas-instances/action.yaml b/xtest/setup-kas-instances/action.yaml
new file mode 100644
index 000000000..f5e420172
--- /dev/null
+++ b/xtest/setup-kas-instances/action.yaml
@@ -0,0 +1,86 @@
+name: setup-kas-instances
+description: >-
+ Start additional KAS instances for multi-KAS / ABAC tests.
+ Uses otdf-local ci start-kas to start all 6 KAS instances
+ (alpha, beta, gamma, delta, km1, km2) and expose their log file paths.
+
+inputs:
+ platform-working-dir:
+ description: Path to the platform checkout directory
+ required: true
+ root-key:
+ description: Root key for KAS instances
+ required: true
+ key-management-supported:
+ description: Enable key management on km1/km2 instances (true/false)
+ required: false
+ default: "false"
+ ec-tdf-enabled:
+ description: Enable EC TDF support
+ required: false
+ default: "true"
+ log-type:
+ description: Log format type
+ required: false
+ default: "json"
+ tests-path:
+ description: Path to the tests repo checkout
+ required: false
+ default: "otdftests"
+
+outputs:
+ kas-alpha-log-file:
+ description: Path to KAS alpha log file
+ value: ${{ steps.start-kas.outputs.kas-alpha-log-file }}
+ kas-beta-log-file:
+ description: Path to KAS beta log file
+ value: ${{ steps.start-kas.outputs.kas-beta-log-file }}
+ kas-gamma-log-file:
+ description: Path to KAS gamma log file
+ value: ${{ steps.start-kas.outputs.kas-gamma-log-file }}
+ kas-delta-log-file:
+ description: Path to KAS delta log file
+ value: ${{ steps.start-kas.outputs.kas-delta-log-file }}
+ kas-km1-log-file:
+ description: Path to KAS km1 log file
+ value: ${{ steps.start-kas.outputs.kas-km1-log-file }}
+ kas-km2-log-file:
+ description: Path to KAS km2 log file
+ value: ${{ steps.start-kas.outputs.kas-km2-log-file }}
+
+runs:
+ using: composite
+ steps:
+ - name: Start KAS instances
+ id: start-kas
+ shell: bash
+ run: |
+ KM_FLAG=""
+ if [[ "$KEY_MANAGEMENT" == "true" ]]; then
+ KM_FLAG="--key-management"
+ else
+ KM_FLAG="--no-key-management"
+ fi
+
+ EC_FLAG=""
+ if [[ "$EC_TDF_ENABLED" == "true" ]]; then
+ EC_FLAG="--ec-tdf-enabled"
+ else
+ EC_FLAG="--no-ec-tdf"
+ fi
+
+ OTDF_LOCAL_DIR="$(cd "$TESTS_PATH" && pwd)/otdf-local"
+
+ uv run --project "$OTDF_LOCAL_DIR" otdf-local ci start-kas \
+ --platform-dir "$(pwd)/$PLATFORM_DIR" \
+ --root-key "$ROOT_KEY" \
+ $EC_FLAG \
+ $KM_FLAG \
+ --log-type "$LOG_TYPE"
+ env:
+ PLATFORM_DIR: ${{ inputs.platform-working-dir }}
+ ROOT_KEY: ${{ inputs.root-key }}
+ KEY_MANAGEMENT: ${{ inputs.key-management-supported }}
+ EC_TDF_ENABLED: ${{ inputs.ec-tdf-enabled }}
+ LOG_TYPE: ${{ inputs.log-type }}
+ TESTS_PATH: ${{ inputs.tests-path }}
diff --git a/xtest/setup-sdk-clients/action.yaml b/xtest/setup-sdk-clients/action.yaml
new file mode 100644
index 000000000..35be35b41
--- /dev/null
+++ b/xtest/setup-sdk-clients/action.yaml
@@ -0,0 +1,159 @@
+name: setup-sdk-clients
+description: >-
+ Configure, cache, patch, and build an SDK CLI for xtest. Wraps setup-cli-tool
+ and adds SDK-appropriate caching, go.mod/java .env fixups, and make builds.
+ Each invocation handles one SDK (go, java, or js).
+
+inputs:
+ sdk:
+ description: "SDK to set up: go, java, or js"
+ required: true
+ version-info:
+ description: JSON-encoded output of otdf-sdk-mgr versions resolve for this SDK
+ required: true
+ tests-path:
+ description: Path to the tests repo checkout
+ required: false
+ default: "otdftests"
+ platform-working-dir:
+ description: >-
+ Platform checkout directory. Used for go-fixup (bridging client go.mod
+ to server shared modules) and detecting platform-embedded otdfctl.
+ required: false
+ platform-heads:
+ description: JSON list of platform tags that are heads (from resolve-versions)
+ required: false
+ default: "[]"
+ platform-tag:
+ description: Current matrix platform-tag value
+ required: false
+ platform-tag-to-sha:
+ description: JSON object mapping platform tags to SHAs
+ required: false
+ default: "{}"
+ otdfctl-source:
+ description: "Resolved otdfctl source: platform or standalone"
+ required: false
+ default: "standalone"
+ otdfctl-dir:
+ description: Absolute path to platform's otdfctl directory
+ required: false
+ otdfctl-sha:
+ description: SHA of the platform otdfctl checkout
+ required: false
+ focus-sdk:
+ description: "SDK focus filter: all, go, java, or js"
+ required: false
+ default: "all"
+ buf-token:
+ description: BUF token for Java proto compilation
+ required: false
+ pat-opentdf:
+ description: PAT for buf HTTPS password (Java SDK build)
+ required: false
+
+outputs:
+ heads:
+ description: JSON list of head tags for this SDK
+ value: ${{ steps.configure.outputs.heads }}
+
+runs:
+ using: composite
+ steps:
+ # Step 1: Configure SDK via setup-cli-tool (checkout/install)
+ - name: Configure ${{ inputs.sdk }}
+ id: configure
+ uses: ./otdftests/xtest/setup-cli-tool
+ with:
+ path: ${{ inputs.tests-path }}/xtest/sdk
+ sdk: ${{ inputs.sdk }}
+ version-info: ${{ inputs.version-info }}
+ platform-otdfctl-dir: ${{ inputs.otdfctl-dir }}
+ platform-otdfctl-sha: ${{ inputs.otdfctl-sha }}
+
+ # Step 2: SDK-appropriate dependency caching
+ - name: Cache npm
+ if: inputs.sdk == 'js' && fromJson(steps.configure.outputs.heads)[0] != null
+ uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2
+ with:
+ path: ~/.npm
+ key: npm-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/js/src/**/package-lock.json', inputs.tests-path)) }}
+ restore-keys: |
+ npm-${{ runner.os }}-
+
+ - name: Cache Go modules
+ if: inputs.sdk == 'go' && fromJson(steps.configure.outputs.heads)[0] != null
+ uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2
+ with:
+ path: |
+ ~/.cache/go-build
+ ~/go/pkg/mod
+ key: go-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/go/src/*/go.sum', inputs.tests-path)) }}
+ restore-keys: |
+ go-${{ runner.os }}-
+
+ - name: Cache Maven repository
+ if: inputs.sdk == 'java' && fromJson(steps.configure.outputs.heads)[0] != null
+ uses: actions/cache@8b402f58fbc84540c8b491a91e594a4576fec3d7 # v5.0.2
+ with:
+ path: ~/.m2/repository
+ key: maven-${{ runner.os }}-${{ hashFiles(format('{0}/xtest/sdk/java/src/**/pom.xml', inputs.tests-path)) }}
+ restore-keys: |
+ maven-${{ runner.os }}-
+
+ # Step 3: SDK-specific fixups
+
+ # Go: Bridge client go.mod to server shared modules (standalone otdfctl only)
+ - name: Go fixup - replace go.mod packages
+ if: >-
+ inputs.sdk == 'go'
+ && steps.configure.outputs.heads != '[]'
+ && inputs.otdfctl-source != 'platform'
+ && inputs.focus-sdk == 'go'
+ && contains(fromJSON(inputs.platform-heads), inputs.platform-tag)
+ && inputs.platform-working-dir != ''
+ shell: bash
+ run: |
+ SDK_MGR_DIR="$(cd "$TESTS_PATH" && pwd)/otdf-sdk-mgr"
+ PLATFORM_DIR_ABS="$(pwd)/${PLATFORM_WORKING_DIR}"
+ BASE_DIR="$(pwd)/${TESTS_PATH}/xtest/sdk/go/src"
+ HEADS='${{ steps.configure.outputs.heads }}'
+ uv run --project "$SDK_MGR_DIR" otdf-sdk-mgr go-fixup \
+ --platform-dir "$PLATFORM_DIR_ABS" \
+ --heads "$HEADS" \
+ "$BASE_DIR"
+ env:
+ PLATFORM_WORKING_DIR: ${{ inputs.platform-working-dir }}
+ TESTS_PATH: ${{ inputs.tests-path }}
+
+ # Java: Create .env files with PLATFORM_BRANCH for head versions
+ - name: Java fixup - create platform branch .env files
+ if: >-
+ inputs.sdk == 'java'
+ && steps.configure.outputs.heads != '[]'
+ && (inputs.focus-sdk == 'go' || inputs.focus-sdk == 'java')
+ && contains(fromJSON(inputs.platform-heads), inputs.platform-tag)
+ shell: bash
+ run: |
+ for row in $(echo "$VERSION_INFO" | jq -c '.[]'); do
+ TAG=$(echo "$row" | jq -r '.tag')
+ HEAD=$(echo "$row" | jq -r '.head')
+ if [[ "$HEAD" == "true" ]]; then
+ echo "Creating .env file for tag: [$TAG]; pointing to platform ref [$PLATFORM_REF]"
+ echo "PLATFORM_BRANCH=$PLATFORM_REF" > "${TESTS_PATH}/xtest/sdk/java/${TAG}.env"
+ fi
+ done
+ env:
+ VERSION_INFO: ${{ inputs.version-info }}
+ PLATFORM_REF: ${{ fromJSON(inputs.platform-tag-to-sha)[inputs.platform-tag] }}
+ TESTS_PATH: ${{ inputs.tests-path }}
+
+ # Step 4: Build the SDK CLI
+ - name: Build ${{ inputs.sdk }} CLI
+ if: fromJson(steps.configure.outputs.heads)[0] != null
+ shell: bash
+ run: make
+ working-directory: ${{ inputs.tests-path }}/xtest/sdk/${{ inputs.sdk }}
+ env:
+ BUF_INPUT_HTTPS_USERNAME: ${{ inputs.sdk == 'java' && 'opentdf-bot' || '' }}
+ BUF_INPUT_HTTPS_PASSWORD: ${{ inputs.sdk == 'java' && inputs.pat-opentdf || '' }}
diff --git a/xtest/setup-test-environment/action.yaml b/xtest/setup-test-environment/action.yaml
new file mode 100644
index 000000000..0f8ce86f3
--- /dev/null
+++ b/xtest/setup-test-environment/action.yaml
@@ -0,0 +1,140 @@
+name: setup-test-environment
+description: >-
+ Detect platform capabilities, extract configuration, and prepare the test
+ environment. Consolidates otdfctl detection, platform version lookup, key
+ management support, root key extraction, multikas support check, and test
+ dependency installation.
+
+inputs:
+ platform-working-dir:
+ description: Platform checkout directory (from start-up-with-containers output)
+ required: true
+ platform-tag:
+ description: Platform version tag under test (matrix value)
+ required: true
+ otdfctl-source-input:
+ description: "User's otdfctl-source preference: auto, standalone, or platform"
+ required: false
+ default: "auto"
+ tests-path:
+ description: Path to the tests repo checkout
+ required: false
+ default: "otdftests"
+
+outputs:
+ otdfctl-source:
+ description: "Resolved otdfctl source: platform or standalone"
+ value: ${{ steps.detect-otdfctl.outputs.otdfctl-source }}
+ otdfctl-dir:
+ description: Absolute path to otdfctl directory (if source=platform)
+ value: ${{ steps.detect-otdfctl.outputs.otdfctl-dir }}
+ otdfctl-sha:
+ description: SHA of the platform checkout (if source=platform)
+ value: ${{ steps.detect-otdfctl.outputs.otdfctl-sha }}
+ platform-version:
+ description: Detected platform version string
+ value: ${{ steps.platform-version.outputs.version }}
+ key-management-supported:
+ description: Whether the platform supports key management (true/false)
+ value: ${{ steps.km-check.outputs.supported }}
+ root-key:
+ description: Root key for KAS instances
+ value: ${{ steps.km-check.outputs.root_key }}
+ multikas-supported:
+ description: Whether multi-KAS is supported (true/false)
+ value: ${{ steps.multikas.outputs.supported }}
+
+runs:
+ using: composite
+ steps:
+ - name: Detect platform-embedded otdfctl
+ id: detect-otdfctl
+ shell: bash
+ run: |
+ if [[ "$OTDFCTL_SOURCE_INPUT" == "auto" || -z "$OTDFCTL_SOURCE_INPUT" ]]; then
+ if [ -d "$PLATFORM_DIR/otdfctl" ] && [ -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then
+ echo "otdfctl found in platform checkout at $PLATFORM_DIR/otdfctl"
+ echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT"
+ echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT"
+ echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+ else
+ echo "otdfctl not found in platform checkout; using standalone repo"
+ echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT"
+ fi
+ elif [[ "$OTDFCTL_SOURCE_INPUT" == "platform" ]]; then
+ if [ -z "$PLATFORM_DIR" ] || [ ! -d "$PLATFORM_DIR/otdfctl" ] || [ ! -f "$PLATFORM_DIR/otdfctl/go.mod" ]; then
+ echo "::error::otdfctl-source=platform requested but ${PLATFORM_DIR:-}/otdfctl does not exist or lacks go.mod"
+ exit 1
+ fi
+ echo "otdfctl-source=platform" >> "$GITHUB_OUTPUT"
+ echo "otdfctl-dir=$(pwd)/$PLATFORM_DIR/otdfctl" >> "$GITHUB_OUTPUT"
+ echo "otdfctl-sha=$(git -C "$PLATFORM_DIR" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+ else
+ echo "otdfctl-source=standalone" >> "$GITHUB_OUTPUT"
+ fi
+ env:
+ OTDFCTL_SOURCE_INPUT: ${{ inputs.otdfctl-source-input }}
+ PLATFORM_DIR: ${{ inputs.platform-working-dir }}
+
+ - name: Lookup platform version
+ id: platform-version
+ shell: bash
+ run: |
+ if ! go run ./service version; then
+ echo "Error: Unable to get platform version; defaulting to tag: [$PLATFORM_TAG]"
+ echo "version=$PLATFORM_TAG" >> "$GITHUB_OUTPUT"
+ exit
+ fi
+ PLATFORM_VERSION=$(go run ./service version 2>&1)
+ echo "version=$PLATFORM_VERSION" >> "$GITHUB_OUTPUT"
+ working-directory: ${{ inputs.platform-working-dir }}
+ env:
+ PLATFORM_TAG: ${{ inputs.platform-tag }}
+
+ - name: Check key management support and prepare root key
+ id: km-check
+ shell: bash
+ run: |
+ OT_CONFIG_FILE="$(pwd)/opentdf.yaml"
+ km_value=$(yq e '.services.kas.preview.key_management' "$OT_CONFIG_FILE" 2>/dev/null || echo "null")
+ case "$km_value" in
+ true|false)
+ echo "supported=true" >> "$GITHUB_OUTPUT"
+ ;;
+ *)
+ echo "supported=false" >> "$GITHUB_OUTPUT"
+ ;;
+ esac
+ existing_root_key=$(yq e '.services.kas.root_key' "$OT_CONFIG_FILE" 2>/dev/null || echo "")
+ if [ -n "$existing_root_key" ] && [ "$existing_root_key" != "null" ]; then
+ echo "Using existing root key from config"
+ echo "root_key=$existing_root_key" >> "$GITHUB_OUTPUT"
+ else
+ echo "Generating a new root key for additional KAS"
+ gen_root_key=$(openssl rand -hex 32)
+ echo "root_key=$gen_root_key" >> "$GITHUB_OUTPUT"
+ fi
+ working-directory: ${{ inputs.platform-working-dir }}
+
+ - name: Check multikas support
+ id: multikas
+ shell: bash
+ run: |
+ if [[ $PLATFORM_TAG == main ]]; then
+ echo "Main supports multikas"
+ echo "supported=true" >> "$GITHUB_OUTPUT"
+ elif awk -F. '{ if ($1 > 0 || ($1 == 0 && $2 > 4)) exit 0; else exit 1; }' <<< "${PLATFORM_VERSION#v}"; then
+ echo "Selected version [$PLATFORM_VERSION] supports multikas"
+ echo "supported=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "At tag [$PLATFORM_TAG], [$PLATFORM_VERSION] probably does not support multikas"
+ echo "supported=false" >> "$GITHUB_OUTPUT"
+ fi
+ env:
+ PLATFORM_TAG: ${{ inputs.platform-tag }}
+ PLATFORM_VERSION: ${{ steps.platform-version.outputs.version }}
+
+ - name: Install test dependencies
+ shell: bash
+ run: uv sync
+ working-directory: ${{ inputs.tests-path }}/xtest