feat(xtest): Allow selecting sdk versions for xtest

.github/workflows/xtest.yml

@@ -82,6 +82,7 @@ jobs:
       go: ${{ steps.version-info.outputs.go-version-info }}
       java: ${{ steps.version-info.outputs.java-version-info }}
       js: ${{ steps.version-info.outputs.js-version-info }}
+      sdk-version-list: ${{ steps.version-info.outputs.sdk-version-list }}
     env:
       PLATFORM_REF: "${{ inputs.platform-ref }}"
       JS_REF: "${{ inputs.js-ref }}"
@@ -185,6 +186,15 @@ jobs:
 
             core.setOutput('all', JSON.stringify(versionData));
 
+            const sdkVersionList = [];
+            for (const [sdkType, refInfo] of Object.entries(versionData)) {
+              if (sdkType === 'platform') continue;
+              for (const { tag, err } of refInfo) {
+                if (!err) sdkVersionList.push(`${sdkType}@${tag}`);
+              }
+            }
+            core.setOutput('sdk-version-list', JSON.stringify(sdkVersionList));
+
             core.summary.addHeading('Versions under Test', 3);
 
             function artifactLink(sdkType, tag, release, head, source) {
@@ -263,12 +273,13 @@ jobs:
       pull-requests: write # Add comments to PRs
     env:
       FOCUS_SDK: ${{ inputs.focus-sdk || 'all' }}
-      ENCRYPT_SDK: ${{ matrix.sdk }}
+      ENCRYPT_SDK: ${{ matrix.sdk-version }}
+      SKIP_RELEASED_PAIRS: ${{ github.event_name != 'workflow_dispatch' && !contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) }}
     strategy:
       fail-fast: false
       matrix:
         platform-tag: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-list) }}
-        sdk: ["go", "java", "js"]
+        sdk-version: ${{ fromJSON(needs.resolve-versions.outputs.sdk-version-list) }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -408,7 +419,7 @@ jobs:
         if: >-
           steps.detect-otdfctl.outputs.otdfctl-source != 'platform'
           && fromJson(steps.configure-go.outputs.heads)[0] != null
-          && env.FOCUS_SDK == 'go'
+          && startsWith(matrix.sdk-version, 'go@')
           && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag)
         env:
           PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }}
@@ -456,7 +467,7 @@ jobs:
       - name: pre-release protocol buffers for java-sdk
         if: >-
           fromJson(steps.configure-java.outputs.heads)[0] != null
-          && (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java')
+          && (startsWith(matrix.sdk-version, 'go@') || startsWith(matrix.sdk-version, 'java@'))
           && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag)
         run: |-
           echo "Replacing .env files for java-sdk..."
@@ -559,7 +570,8 @@ jobs:
       ######## RUN THE TESTS #############
       - name: Run legacy decryption tests
         run: |-
-          uv run pytest -n auto --dist worksteal --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_legacy.py
+          skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "")
+          uv run pytest -n auto --dist worksteal --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-decrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" $skip_flag test_legacy.py
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -568,7 +580,8 @@ jobs:
       - name: Run all standard xtests
         if: ${{ env.FOCUS_SDK == 'all' }}
         run: |-
-          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v test_tdfs.py test_policytypes.py test_pqc.py
+          skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "")
+          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v $skip_flag test_tdfs.py test_policytypes.py test_pqc.py
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -578,7 +591,8 @@ jobs:
       - name: Run xtests focusing on a specific SDK
         if: ${{ env.FOCUS_SDK != 'all' }}
         run: |-
-          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_tdfs.py test_policytypes.py test_pqc.py
+          skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "")
+          uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" $skip_flag test_tdfs.py test_policytypes.py test_pqc.py
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -673,18 +687,16 @@ jobs:
 
       - name: Run attribute based configuration tests
         if: ${{ steps.multikas.outputs.supported == 'true' }}
-        run: >-
-          uv run pytest
-          -ra
-          -v
-          --numprocesses auto
-          --dist loadscope
-          --html test-results/attributes-${FOCUS_SDK}-${PLATFORM_TAG}.html
-          --self-contained-html
-          --audit-log-dir test-results/audit-logs
-          --sdks-encrypt "${ENCRYPT_SDK}"
-          --focus "$FOCUS_SDK"
-          test_abac.py
+        run: |-
+          skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "")
+          uv run pytest -ra -v --numprocesses auto --dist loadscope \
+            --html test-results/attributes-${FOCUS_SDK}-${PLATFORM_TAG}.html \
+            --self-contained-html \
+            --audit-log-dir test-results/audit-logs \
+            --sdks-encrypt "${ENCRYPT_SDK}" \
+            --focus "$FOCUS_SDK" \
+            $skip_flag \
+            test_abac.py
         working-directory: otdftests/xtest
         env:
           PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}"
@@ -697,19 +709,24 @@ jobs:
           KAS_KM1_LOG_FILE: "../../${{ steps.kas-km1.outputs.log-file }}"
           KAS_KM2_LOG_FILE: "../../${{ steps.kas-km2.outputs.log-file }}"
 
+      - name: Sanitize sdk-version for artifact name
+        id: artifact-name
+        if: success() || failure()
+        run: echo "sdk_version=${ENCRYPT_SDK//\//-}" >> "$GITHUB_OUTPUT"
+
       - name: Upload artifact
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
         id: upload-artifact
         if: success() || failure()
         with:
-          name: ${{ job.status == 'success' && '✅' || job.status == 'failure' && '❌' }} ${{ matrix.sdk }}-${{matrix.platform-tag}}
+          name: ${{ job.status == 'success' && '✅' || job.status == 'failure' && '❌' }} ${{ steps.artifact-name.outputs.sdk_version }}-${{ matrix.platform-tag }}
           path: otdftests/xtest/test-results/*.html
 
       - name: Upload audit logs on failure
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
         if: failure()
         with:
-          name: audit-logs-${{ matrix.sdk }}-${{ matrix.platform-tag }}
+          name: audit-logs-${{ steps.artifact-name.outputs.sdk_version }}-${{ matrix.platform-tag }}
           path: otdftests/xtest/test-results/audit-logs/*.log
           if-no-files-found: ignore
 
@@ -765,9 +782,9 @@ jobs:
       contents: read
     steps:
       - name: Assert all matrix jobs passed
-        if: ${{ needs.xct.result == 'failure' || needs.xct.result == 'cancelled' }}
+        if: ${{ needs.xct.result == 'failure' || needs.xct.result == 'cancelled' || needs.xct.result == 'skipped' }}
         run: |-
-          echo "xct matrix had failures (overall result: ${XCT_RESULT}). Marking xtest failed." >> "$GITHUB_STEP_SUMMARY"
+          echo "xct matrix failed or was skipped (overall result: ${XCT_RESULT}). Marking xtest failed." >> "$GITHUB_STEP_SUMMARY"
           exit 1
         env:
           XCT_RESULT: ${{ needs.xct.result }}

xtest/audit_logs.py

@@ -1711,10 +1711,10 @@ def _raise_assertion_error(
 
         if matching:
             context.append("Matching logs:")
-            for log in matching[:10]:
-                context.append(
-                    f"  [{log.timestamp}] {log.service_name}: {log.raw_line}"
-                )
+            context.extend(
+                f"  [{log.timestamp}] {log.service_name}: {log.raw_line}"
+                for log in matching[:10]
+            )
             if len(matching) > 10:
                 context.append(f"  ... and {len(matching) - 10} more")
             context.append("")
@@ -1734,10 +1734,10 @@ def _raise_assertion_error(
             context.append(
                 f"Logs before timeout (last {len(recent_logs)} of {len(all_logs)}):"
             )
-            for log in recent_logs:
-                context.append(
-                    f"  [{log.timestamp}] {log.service_name}: {log.raw_line}"
-                )
+            context.extend(
+                f"  [{log.timestamp}] {log.service_name}: {log.raw_line}"
+                for log in recent_logs
+            )
 
         # Show timeout marker
         if timeout_time:
@@ -1751,10 +1751,10 @@ def _raise_assertion_error(
             context.append(
                 f"Logs AFTER timeout ({len(late_to_show)} of {len(late_logs)} late arrivals):"
             )
-            for log in late_to_show:
-                context.append(
-                    f"  [{log.timestamp}] {log.service_name}: {log.raw_line}"
-                )
+            context.extend(
+                f"  [{log.timestamp}] {log.service_name}: {log.raw_line}"
+                for log in late_to_show
+            )
             if len(late_logs) > 10:
                 context.append(f"  ... and {len(late_logs) - 10} more late arrivals")
             context.append("")

xtest/conftest.py

@@ -60,6 +60,22 @@ def is_a(v: str) -> typing.Any:
     return is_a
 
 
+def sdk_spec_type(v: str) -> str:
+    """Validate a whitespace-separated list of SDK specifiers: 'go', 'go@*', 'go@main java@v1.2.0', etc."""
+    specs = v.split()
+    if not specs:
+        raise ValueError("At least one SDK specifier is required")
+    for spec in specs:
+        parts = spec.split("@", 1)
+        if not tdfs.is_sdk_type(parts[0]):
+            raise ValueError(f"Invalid SDK type: {parts[0]!r}")
+        if len(parts) == 2 and not parts[1]:
+            raise ValueError(
+                f"Empty version in SDK specifier {spec!r}; use e.g. go@main, go@v0.18.0, go@*"
+            )
+    return v
+
+
 def pytest_addoption(parser: pytest.Parser):
     """Add custom CLI options for pytest."""
     parser.addoption(
@@ -92,20 +108,25 @@ def pytest_addoption(parser: pytest.Parser):
         action="store_true",
         help="disable automatic KAS audit log collection",
     )
+    parser.addoption(
+        "--skip-released-pairs",
+        action="store_true",
+        help="skip round-trip tests where all SDKs are released artifacts",
+    )
     parser.addoption(
         "--sdks",
-        help=f"select which sdks to run by default, unless overridden, one or more of {englist(typing.get_args(tdfs.sdk_type))}",
-        type=is_type_or_list_of_types(tdfs.sdk_type),
+        help=f"select which sdks to run by default, unless overridden; one or more of {englist(typing.get_args(tdfs.sdk_type))}, optionally version-qualified (e.g. go@main, go@v0.18.0, go@*)",
+        type=sdk_spec_type,
     )
     parser.addoption(
         "--sdks-decrypt",
-        help="select which sdks to run for decrypt only",
-        type=is_type_or_list_of_types(tdfs.sdk_type),
+        help="select which sdks to run for decrypt only; accepts same format as --sdks",
+        type=sdk_spec_type,
     )
     parser.addoption(
         "--sdks-encrypt",
-        help="select which sdks to run for encrypt only",
-        type=is_type_or_list_of_types(tdfs.sdk_type),
+        help="select which sdks to run for encrypt only; accepts same format as --sdks",
+        type=sdk_spec_type,
     )
 
 
@@ -139,44 +160,36 @@ def list_opt(name: str, t: typing.Any) -> list[str]:
                 raise ValueError(f"Invalid value for {name}: {i}, must be one of {ttt}")
         return a
 
-    def defaulted_list_opt[T](
-        names: list[str], t: typing.Any, default: list[T]
-    ) -> list[T]:
+    def sdk_specs_opt(names: list[str]) -> list[str]:
+        """Return SDK specifier tokens from the first matching option, or all sdk types."""
         for name in names:
             v = metafunc.config.getoption(name)
             if v:
-                return cast(list[T], list_opt(name, t))
-        return default
+                return v.split()
+        return list(typing.get_args(tdfs.sdk_type))
 
     subject_sdks: set[tdfs.SDK] = set()
 
     if "encrypt_sdk" in metafunc.fixturenames:
-        encrypt_sdks: list[tdfs.sdk_type] = []
-        encrypt_sdks = defaulted_list_opt(
-            ["--sdks-encrypt", "--sdks"],
-            tdfs.sdk_type,
-            list(typing.get_args(tdfs.sdk_type)),
-        )
-        # convert list of sdk_type to list of SDK objects
-        e_sdks = [
-            v
-            for sdks in [tdfs.all_versions_of(sdk) for sdk in encrypt_sdks]
-            for v in sdks
-        ]
+        try:
+            e_sdks = [
+                sdk
+                for spec in sdk_specs_opt(["--sdks-encrypt", "--sdks"])
+                for sdk in tdfs.parse_sdk_spec(spec)
+            ]
+        except (FileNotFoundError, ValueError) as e:
+            raise pytest.UsageError(str(e)) from e
         metafunc.parametrize("encrypt_sdk", e_sdks, ids=[str(x) for x in e_sdks])
         subject_sdks |= set(e_sdks)
     if "decrypt_sdk" in metafunc.fixturenames:
-        decrypt_sdks: list[tdfs.sdk_type] = []
-        decrypt_sdks = defaulted_list_opt(
-            ["--sdks-decrypt", "--sdks"],
-            tdfs.sdk_type,
-            list(typing.get_args(tdfs.sdk_type)),
-        )
-        d_sdks = [
-            v
-            for sdks in [tdfs.all_versions_of(sdk) for sdk in decrypt_sdks]
-            for v in sdks
-        ]
+        try:
+            d_sdks = [
+                sdk
+                for spec in sdk_specs_opt(["--sdks-decrypt", "--sdks"])
+                for sdk in tdfs.parse_sdk_spec(spec)
+            ]
+        except (FileNotFoundError, ValueError) as e:
+            raise pytest.UsageError(str(e)) from e
         metafunc.parametrize("decrypt_sdk", d_sdks, ids=[str(x) for x in d_sdks])
         subject_sdks |= set(d_sdks)
 
@@ -203,6 +216,18 @@ def defaulted_list_opt[T](
         metafunc.parametrize("container", containers)
 
 
+def pytest_runtest_setup(item: pytest.Item):
+    if not item.config.getoption("--skip-released-pairs", default=False):
+        return
+    params = getattr(item, "callspec", None)
+    if params is None:
+        return
+    e = params.params.get("encrypt_sdk")
+    d = params.params.get("decrypt_sdk")
+    if e is not None and d is not None and e.is_released() and d.is_released():
+        pytest.skip(f"released-only pair ({e} × {d})")
+
+
 # Core fixtures
 @pytest.fixture(scope="session")
 def pt_file(tmp_dir: Path, size: str) -> Path:

xtest/pyproject.toml

@@ -70,6 +70,7 @@ select = [
     "B",      # flake8-bugbear
     "C4",     # flake8-comprehensions
     "UP",     # pyupgrade
+    "PERF",   # Perflint
 ]
 ignore = [
     "E501",   # line too long (handled by formatter)
@@ -87,6 +88,8 @@ indent-style = "space"
 [tool.pyright]
 pythonVersion = "3.14"
 typeCheckingMode = "standard"
+venvPath = "."
+venv = ".venv"
 include = ["."]
 exclude = ["**/__pycache__", ".venv", ".ruff_cache", ".pytest_cache", "sdk", "tmp", "golden"]
 reportMissingImports = true