diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index 14c15019..ed89c4b0 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -82,6 +82,7 @@ jobs: go: ${{ steps.version-info.outputs.go-version-info }} java: ${{ steps.version-info.outputs.java-version-info }} js: ${{ steps.version-info.outputs.js-version-info }} + sdk-version-list: ${{ steps.version-info.outputs.sdk-version-list }} env: PLATFORM_REF: "${{ inputs.platform-ref }}" JS_REF: "${{ inputs.js-ref }}" @@ -185,6 +186,15 @@ jobs: core.setOutput('all', JSON.stringify(versionData)); + const sdkVersionList = []; + for (const [sdkType, refInfo] of Object.entries(versionData)) { + if (sdkType === 'platform') continue; + for (const { tag, err } of refInfo) { + if (!err) sdkVersionList.push(`${sdkType}@${tag}`); + } + } + core.setOutput('sdk-version-list', JSON.stringify(sdkVersionList)); + core.summary.addHeading('Versions under Test', 3); function artifactLink(sdkType, tag, release, head, source) { @@ -263,12 +273,13 @@ jobs: pull-requests: write # Add comments to PRs env: FOCUS_SDK: ${{ inputs.focus-sdk || 'all' }} - ENCRYPT_SDK: ${{ matrix.sdk }} + ENCRYPT_SDK: ${{ matrix.sdk-version }} + SKIP_RELEASED_PAIRS: ${{ github.event_name != 'workflow_dispatch' && !contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) }} strategy: fail-fast: false matrix: platform-tag: ${{ fromJSON(needs.resolve-versions.outputs.platform-tag-list) }} - sdk: ["go", "java", "js"] + sdk-version: ${{ fromJSON(needs.resolve-versions.outputs.sdk-version-list) }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -408,7 +419,7 @@ jobs: if: >- steps.detect-otdfctl.outputs.otdfctl-source != 'platform' && fromJson(steps.configure-go.outputs.heads)[0] != null - && env.FOCUS_SDK == 'go' + && startsWith(matrix.sdk-version, 'go@') && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) env: PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} @@ -456,7 +467,7 @@ jobs: - name: pre-release protocol buffers for java-sdk if: >- fromJson(steps.configure-java.outputs.heads)[0] != null - && (env.FOCUS_SDK == 'go' || env.FOCUS_SDK == 'java') + && (startsWith(matrix.sdk-version, 'go@') || startsWith(matrix.sdk-version, 'java@')) && contains(fromJSON(needs.resolve-versions.outputs.heads), matrix.platform-tag) run: |- echo "Replacing .env files for java-sdk..." @@ -559,7 +570,8 @@ jobs: ######## RUN THE TESTS ############# - name: Run legacy decryption tests run: |- - uv run pytest -n auto --dist worksteal --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_legacy.py + skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "") + uv run pytest -n auto --dist worksteal --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-decrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" $skip_flag test_legacy.py working-directory: otdftests/xtest env: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" @@ -568,7 +580,8 @@ jobs: - name: Run all standard xtests if: ${{ env.FOCUS_SDK == 'all' }} run: |- - uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v test_tdfs.py test_policytypes.py test_pqc.py + skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "") + uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v $skip_flag test_tdfs.py test_policytypes.py test_pqc.py working-directory: otdftests/xtest env: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" @@ -578,7 +591,8 @@ jobs: - name: Run xtests focusing on a specific SDK if: ${{ env.FOCUS_SDK != 'all' }} run: |- - uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" test_tdfs.py test_policytypes.py test_pqc.py + skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "") + uv run pytest -n auto --dist loadscope --html=test-results/sdk-${FOCUS_SDK}-${PLATFORM_TAG}.html --self-contained-html --sdks-encrypt "${ENCRYPT_SDK}" -ra -v --focus "$FOCUS_SDK" $skip_flag test_tdfs.py test_policytypes.py test_pqc.py working-directory: otdftests/xtest env: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" @@ -673,18 +687,16 @@ jobs: - name: Run attribute based configuration tests if: ${{ steps.multikas.outputs.supported == 'true' }} - run: >- - uv run pytest - -ra - -v - --numprocesses auto - --dist loadscope - --html test-results/attributes-${FOCUS_SDK}-${PLATFORM_TAG}.html - --self-contained-html - --audit-log-dir test-results/audit-logs - --sdks-encrypt "${ENCRYPT_SDK}" - --focus "$FOCUS_SDK" - test_abac.py + run: |- + skip_flag=$([[ "$SKIP_RELEASED_PAIRS" == "true" ]] && echo "--skip-released-pairs" || echo "") + uv run pytest -ra -v --numprocesses auto --dist loadscope \ + --html test-results/attributes-${FOCUS_SDK}-${PLATFORM_TAG}.html \ + --self-contained-html \ + --audit-log-dir test-results/audit-logs \ + --sdks-encrypt "${ENCRYPT_SDK}" \ + --focus "$FOCUS_SDK" \ + $skip_flag \ + test_abac.py working-directory: otdftests/xtest env: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" @@ -697,19 +709,24 @@ jobs: KAS_KM1_LOG_FILE: "../../${{ steps.kas-km1.outputs.log-file }}" KAS_KM2_LOG_FILE: "../../${{ steps.kas-km2.outputs.log-file }}" + - name: Sanitize sdk-version for artifact name + id: artifact-name + if: success() || failure() + run: echo "sdk_version=${ENCRYPT_SDK//\//-}" >> "$GITHUB_OUTPUT" + - name: Upload artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 id: upload-artifact if: success() || failure() with: - name: ${{ job.status == 'success' && '✅' || job.status == 'failure' && '❌' }} ${{ matrix.sdk }}-${{matrix.platform-tag}} + name: ${{ job.status == 'success' && '✅' || job.status == 'failure' && '❌' }} ${{ steps.artifact-name.outputs.sdk_version }}-${{ matrix.platform-tag }} path: otdftests/xtest/test-results/*.html - name: Upload audit logs on failure uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: failure() with: - name: audit-logs-${{ matrix.sdk }}-${{ matrix.platform-tag }} + name: audit-logs-${{ steps.artifact-name.outputs.sdk_version }}-${{ matrix.platform-tag }} path: otdftests/xtest/test-results/audit-logs/*.log if-no-files-found: ignore @@ -765,9 +782,9 @@ jobs: contents: read steps: - name: Assert all matrix jobs passed - if: ${{ needs.xct.result == 'failure' || needs.xct.result == 'cancelled' }} + if: ${{ needs.xct.result == 'failure' || needs.xct.result == 'cancelled' || needs.xct.result == 'skipped' }} run: |- - echo "xct matrix had failures (overall result: ${XCT_RESULT}). Marking xtest failed." >> "$GITHUB_STEP_SUMMARY" + echo "xct matrix failed or was skipped (overall result: ${XCT_RESULT}). Marking xtest failed." >> "$GITHUB_STEP_SUMMARY" exit 1 env: XCT_RESULT: ${{ needs.xct.result }} diff --git a/xtest/audit_logs.py b/xtest/audit_logs.py index 51f3f292..c894110c 100644 --- a/xtest/audit_logs.py +++ b/xtest/audit_logs.py @@ -1711,10 +1711,10 @@ def _raise_assertion_error( if matching: context.append("Matching logs:") - for log in matching[:10]: - context.append( - f" [{log.timestamp}] {log.service_name}: {log.raw_line}" - ) + context.extend( + f" [{log.timestamp}] {log.service_name}: {log.raw_line}" + for log in matching[:10] + ) if len(matching) > 10: context.append(f" ... and {len(matching) - 10} more") context.append("") @@ -1734,10 +1734,10 @@ def _raise_assertion_error( context.append( f"Logs before timeout (last {len(recent_logs)} of {len(all_logs)}):" ) - for log in recent_logs: - context.append( - f" [{log.timestamp}] {log.service_name}: {log.raw_line}" - ) + context.extend( + f" [{log.timestamp}] {log.service_name}: {log.raw_line}" + for log in recent_logs + ) # Show timeout marker if timeout_time: @@ -1751,10 +1751,10 @@ def _raise_assertion_error( context.append( f"Logs AFTER timeout ({len(late_to_show)} of {len(late_logs)} late arrivals):" ) - for log in late_to_show: - context.append( - f" [{log.timestamp}] {log.service_name}: {log.raw_line}" - ) + context.extend( + f" [{log.timestamp}] {log.service_name}: {log.raw_line}" + for log in late_to_show + ) if len(late_logs) > 10: context.append(f" ... and {len(late_logs) - 10} more late arrivals") context.append("") diff --git a/xtest/conftest.py b/xtest/conftest.py index ce6a7131..d5cc584d 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -60,6 +60,22 @@ def is_a(v: str) -> typing.Any: return is_a +def sdk_spec_type(v: str) -> str: + """Validate a whitespace-separated list of SDK specifiers: 'go', 'go@*', 'go@main java@v1.2.0', etc.""" + specs = v.split() + if not specs: + raise ValueError("At least one SDK specifier is required") + for spec in specs: + parts = spec.split("@", 1) + if not tdfs.is_sdk_type(parts[0]): + raise ValueError(f"Invalid SDK type: {parts[0]!r}") + if len(parts) == 2 and not parts[1]: + raise ValueError( + f"Empty version in SDK specifier {spec!r}; use e.g. go@main, go@v0.18.0, go@*" + ) + return v + + def pytest_addoption(parser: pytest.Parser): """Add custom CLI options for pytest.""" parser.addoption( @@ -92,20 +108,25 @@ def pytest_addoption(parser: pytest.Parser): action="store_true", help="disable automatic KAS audit log collection", ) + parser.addoption( + "--skip-released-pairs", + action="store_true", + help="skip round-trip tests where all SDKs are released artifacts", + ) parser.addoption( "--sdks", - help=f"select which sdks to run by default, unless overridden, one or more of {englist(typing.get_args(tdfs.sdk_type))}", - type=is_type_or_list_of_types(tdfs.sdk_type), + help=f"select which sdks to run by default, unless overridden; one or more of {englist(typing.get_args(tdfs.sdk_type))}, optionally version-qualified (e.g. go@main, go@v0.18.0, go@*)", + type=sdk_spec_type, ) parser.addoption( "--sdks-decrypt", - help="select which sdks to run for decrypt only", - type=is_type_or_list_of_types(tdfs.sdk_type), + help="select which sdks to run for decrypt only; accepts same format as --sdks", + type=sdk_spec_type, ) parser.addoption( "--sdks-encrypt", - help="select which sdks to run for encrypt only", - type=is_type_or_list_of_types(tdfs.sdk_type), + help="select which sdks to run for encrypt only; accepts same format as --sdks", + type=sdk_spec_type, ) @@ -139,44 +160,36 @@ def list_opt(name: str, t: typing.Any) -> list[str]: raise ValueError(f"Invalid value for {name}: {i}, must be one of {ttt}") return a - def defaulted_list_opt[T]( - names: list[str], t: typing.Any, default: list[T] - ) -> list[T]: + def sdk_specs_opt(names: list[str]) -> list[str]: + """Return SDK specifier tokens from the first matching option, or all sdk types.""" for name in names: v = metafunc.config.getoption(name) if v: - return cast(list[T], list_opt(name, t)) - return default + return v.split() + return list(typing.get_args(tdfs.sdk_type)) subject_sdks: set[tdfs.SDK] = set() if "encrypt_sdk" in metafunc.fixturenames: - encrypt_sdks: list[tdfs.sdk_type] = [] - encrypt_sdks = defaulted_list_opt( - ["--sdks-encrypt", "--sdks"], - tdfs.sdk_type, - list(typing.get_args(tdfs.sdk_type)), - ) - # convert list of sdk_type to list of SDK objects - e_sdks = [ - v - for sdks in [tdfs.all_versions_of(sdk) for sdk in encrypt_sdks] - for v in sdks - ] + try: + e_sdks = [ + sdk + for spec in sdk_specs_opt(["--sdks-encrypt", "--sdks"]) + for sdk in tdfs.parse_sdk_spec(spec) + ] + except (FileNotFoundError, ValueError) as e: + raise pytest.UsageError(str(e)) from e metafunc.parametrize("encrypt_sdk", e_sdks, ids=[str(x) for x in e_sdks]) subject_sdks |= set(e_sdks) if "decrypt_sdk" in metafunc.fixturenames: - decrypt_sdks: list[tdfs.sdk_type] = [] - decrypt_sdks = defaulted_list_opt( - ["--sdks-decrypt", "--sdks"], - tdfs.sdk_type, - list(typing.get_args(tdfs.sdk_type)), - ) - d_sdks = [ - v - for sdks in [tdfs.all_versions_of(sdk) for sdk in decrypt_sdks] - for v in sdks - ] + try: + d_sdks = [ + sdk + for spec in sdk_specs_opt(["--sdks-decrypt", "--sdks"]) + for sdk in tdfs.parse_sdk_spec(spec) + ] + except (FileNotFoundError, ValueError) as e: + raise pytest.UsageError(str(e)) from e metafunc.parametrize("decrypt_sdk", d_sdks, ids=[str(x) for x in d_sdks]) subject_sdks |= set(d_sdks) @@ -203,6 +216,18 @@ def defaulted_list_opt[T]( metafunc.parametrize("container", containers) +def pytest_runtest_setup(item: pytest.Item): + if not item.config.getoption("--skip-released-pairs", default=False): + return + params = getattr(item, "callspec", None) + if params is None: + return + e = params.params.get("encrypt_sdk") + d = params.params.get("decrypt_sdk") + if e is not None and d is not None and e.is_released() and d.is_released(): + pytest.skip(f"released-only pair ({e} × {d})") + + # Core fixtures @pytest.fixture(scope="session") def pt_file(tmp_dir: Path, size: str) -> Path: diff --git a/xtest/pyproject.toml b/xtest/pyproject.toml index 3e7b6d08..7ff2d9ec 100644 --- a/xtest/pyproject.toml +++ b/xtest/pyproject.toml @@ -70,6 +70,7 @@ select = [ "B", # flake8-bugbear "C4", # flake8-comprehensions "UP", # pyupgrade + "PERF", # Perflint ] ignore = [ "E501", # line too long (handled by formatter) @@ -87,6 +88,8 @@ indent-style = "space" [tool.pyright] pythonVersion = "3.14" typeCheckingMode = "standard" +venvPath = "." +venv = ".venv" include = ["."] exclude = ["**/__pycache__", ".venv", ".ruff_cache", ".pytest_cache", "sdk", "tmp", "golden"] reportMissingImports = true diff --git a/xtest/tdfs.py b/xtest/tdfs.py index 0c4c9c61..4501604c 100644 --- a/xtest/tdfs.py +++ b/xtest/tdfs.py @@ -8,7 +8,7 @@ import zipfile from collections.abc import Callable from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, TypeIs, get_args import jsonschema import pytest @@ -23,6 +23,11 @@ sdk_type = Literal["go", "java", "js"] + +def is_sdk_type(val: str) -> TypeIs[sdk_type]: + return val in get_args(sdk_type) + + focus_type = Literal[sdk_type, "all"] container_type = Literal[ @@ -354,6 +359,7 @@ def simple_container(container: container_type) -> container_type: class SDK: sdk: sdk_type + version: str _supports: dict[feature_type, bool] def __init__(self, sdk: sdk_type, version: str = "main"): @@ -378,6 +384,13 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: return hash((self.sdk, self.version)) + def is_released(self) -> bool: + return bool( + re.fullmatch( + r"(?:sdk/)?v?\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?", self.version + ) + ) + def encrypt( self, pt_file: Path, @@ -505,14 +518,33 @@ def _uncached_supports(self, feature: feature_type) -> bool: def all_versions_of(sdk: sdk_type) -> list[SDK]: - versions: list[SDK] = [] sdk_path = os.path.join("sdk", sdk, "dist") if not os.path.isdir(sdk_path): return [] - for version in os.listdir(sdk_path): - if os.path.isdir(os.path.join(sdk_path, version)): - versions.append(SDK(sdk, version)) - return versions + return [ + SDK(sdk, version) + for version in os.listdir(sdk_path) + if os.path.isdir(os.path.join(sdk_path, version)) + ] + + +def parse_sdk_spec(spec: str) -> list[SDK]: + """Parse an SDK specifier into SDK objects. + + Supports: + - "go" or "go@*" → all versions in sdk/go/dist/ + - "go@main" or "go@v0.18.0" → only that specific version + """ + if "@" in spec: + sdk_type_val, version = spec.split("@", 1) + if not is_sdk_type(sdk_type_val): + raise ValueError(f"Unknown SDK type: {sdk_type_val!r}") + if version == "*": + return all_versions_of(sdk_type_val) + return [SDK(sdk_type_val, version)] + if not is_sdk_type(spec): + raise ValueError(f"Unknown SDK type: {spec!r}") + return all_versions_of(spec) def skip_if_unsupported(sdk: SDK, *features: feature_type):