diff --git a/otdf-local/src/otdf_local/cli.py b/otdf-local/src/otdf_local/cli.py index d8e3597ff..8d793cc0c 100644 --- a/otdf-local/src/otdf_local/cli.py +++ b/otdf-local/src/otdf_local/cli.py @@ -30,6 +30,7 @@ format_status, print_error, print_info, + print_json, print_success, print_warning, status_spinner, @@ -112,10 +113,26 @@ def up( start_platform = "platform" in service_list start_kas = "kas" in service_list + # Step 0: Ensure temporary keys exist + from otdf_local.utils.keys import ensure_all_temp_keys + + print_info("Checking temporary keys...") + try: + generated = ensure_all_temp_keys( + settings.platform_dir, + settings.keys_dir, + compose_file=settings.docker_compose_file, + ) + if generated: + print_success("Generated missing temporary keys") + except Exception as e: + print_error(f"Failed to generate temporary keys: {e}") + raise typer.Exit(1) from e + # Step 1: Start Docker services if start_docker: print_info("Starting Docker services (Keycloak, PostgreSQL)...") - docker = get_docker_service(settings) + docker = get_docker_service(settings, keys_dir=settings.keys_dir) if not docker.start(): print_error("Failed to start Docker services") raise typer.Exit(1) @@ -274,7 +291,7 @@ def list_services( if json_output: output = [info.to_dict() for info in all_info] - console.print_json(json.dumps(output)) + print_json(json.dumps(output)) return # Table output @@ -611,7 +628,7 @@ def env( # Output in requested format if format == "json": - console.print_json(json.dumps(env_vars, indent=2)) + print_json(json.dumps(env_vars, indent=2)) else: # Shell export format - use plain print to avoid line wrapping for key, value in env_vars.items(): @@ -621,5 +638,91 @@ def env( print(f"export {key}='{escaped_value}'", file=sys.stdout) +@app.command() +def configure( + feature: Annotated[ + str | None, + typer.Argument(help="Feature name to configure (e.g. ec-wrap, key-management)"), + ] = None, + enable: Annotated[ + bool | None, + typer.Option("--enable/--disable", help="Enable or disable the feature"), + ] = None, +) -> None: + """Configure platform feature flags. + + Without arguments, lists all available features and their current state. + With a feature name alone, shows the current state of that feature. + With --enable or --disable, updates the setting persistently. + + Changes survive restarts. If the platform config already exists it is + updated immediately; run 'otdf-local restart platform' to pick up the + change in a running environment. + + Examples: + + otdf-local configure # list all features + + otdf-local configure ec-wrap # show ec-wrap state + + otdf-local configure ec-wrap --enable # enable EC-wrapped TDF + + otdf-local configure ec-wrap --disable # disable EC-wrapped TDF + """ + from rich.table import Table + + from otdf_local.config.overrides import FEATURES, apply_overrides, load_overrides, save_overrides + from otdf_local.utils.yaml import load_yaml, save_yaml + + settings = get_settings() + overrides = load_overrides(settings.xtest_root) + + if feature is None: + # List all features with their current state + table = Table(title="Platform Feature Flags") + table.add_column("Feature", style="cyan") + table.add_column("Description") + table.add_column("State") + for name, (_, description) in FEATURES.items(): + if name in overrides: + state = "[green]enabled[/green]" if overrides[name] else "[red]disabled[/red]" + else: + state = "[dim]default[/dim]" + table.add_row(name, description, state) + console.print(table) + return + + if feature not in FEATURES: + print_error(f"Unknown feature: {feature}") + print_info(f"Valid features: {', '.join(FEATURES)}") + raise typer.Exit(1) + + if enable is None: + # Show current state of this feature + if feature in overrides: + state = "enabled" if overrides[feature] else "disabled" + console.print(f"{feature}: {state} (override)") + else: + console.print(f"{feature}: default (no override set)") + return + + # Save override + overrides[feature] = enable + save_overrides(settings.xtest_root, overrides) + + # Apply immediately to live config if it already exists + platform_config = settings.platform_config + if platform_config.exists(): + data = load_yaml(platform_config) + apply_overrides(data, {feature: enable}) + save_yaml(platform_config, data) + action = "enabled" if enable else "disabled" + print_success(f"{feature} {action} (applied to existing config)") + print_info("Run 'otdf-local restart platform' to apply to a running environment") + else: + action = "enabled" if enable else "disabled" + print_success(f"{feature} {action} (will apply on next start)") + + if __name__ == "__main__": app() diff --git a/otdf-local/src/otdf_local/config/overrides.py b/otdf-local/src/otdf_local/config/overrides.py new file mode 100644 index 000000000..dbd5161cb --- /dev/null +++ b/otdf-local/src/otdf_local/config/overrides.py @@ -0,0 +1,47 @@ +"""User-specified feature overrides for platform config.""" + +from pathlib import Path +from typing import Any + +from otdf_local.utils.yaml import load_yaml, save_yaml, set_nested + +# Maps CLI feature name -> (yaml_path, description) +FEATURES: dict[str, tuple[str, str]] = { + "ec-wrap": ( + "services.kas.preview.ec_tdf_enabled", + "EC-wrapped TDF support (ML-KEM / X-Wing hybrid)", + ), + "key-management": ( + "services.kas.preview.key_management", + "Key management service", + ), +} + + +def _overrides_path(xtest_root: Path) -> Path: + """Path to feature overrides file. Lives in tmp/ but outside config/ so it survives --clean.""" + return xtest_root / "tmp" / "feature-overrides.yaml" + + +def load_overrides(xtest_root: Path) -> dict[str, bool]: + """Load current feature overrides. Returns empty dict if none set.""" + path = _overrides_path(xtest_root) + if not path.exists(): + return {} + data = load_yaml(path) + return dict(data) if data else {} + + +def save_overrides(xtest_root: Path, overrides: dict[str, bool]) -> None: + """Persist feature overrides to disk.""" + path = _overrides_path(xtest_root) + path.parent.mkdir(parents=True, exist_ok=True) + save_yaml(path, overrides) + + +def apply_overrides(config_data: Any, overrides: dict[str, bool]) -> None: + """Apply feature overrides to a loaded YAML config object (in-place).""" + for feature, enabled in overrides.items(): + if feature in FEATURES: + yaml_path, _ = FEATURES[feature] + set_nested(config_data, yaml_path, enabled) diff --git a/otdf-local/src/otdf_local/config/settings.py b/otdf-local/src/otdf_local/config/settings.py index 96a4c20e8..cc31735f3 100644 --- a/otdf-local/src/otdf_local/config/settings.py +++ b/otdf-local/src/otdf_local/config/settings.py @@ -3,7 +3,9 @@ from functools import lru_cache from pathlib import Path -from pydantic import Field +import warnings + +from pydantic import Field, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict from otdf_local.config.ports import Ports @@ -95,6 +97,22 @@ class Settings(BaseSettings): default_factory=lambda: _find_platform_dir(_find_xtest_root()) ) + @field_validator("platform_dir") + @classmethod + def _validate_platform_dir(cls, v: Path) -> Path: + v = v.resolve() + if not v.exists(): + raise ValueError(f"platform_dir does not exist: {v}") + if not (v / "service").is_dir(): + raise ValueError(f"platform_dir {v} missing service/ directory") + if not (v / "opentdf-dev.yaml").exists(): + warnings.warn( + f"platform_dir {v} missing opentdf-dev.yaml — " + "config generation will create it from template", + stacklevel=2, + ) + return v + @property def logs_dir(self) -> Path: """Logs directory.""" @@ -112,13 +130,13 @@ def config_dir(self) -> Path: @property def platform_config(self) -> Path: - """Platform config file path.""" - return self.platform_dir / "opentdf-dev.yaml" + """Platform config file path (gitignored, generated locally).""" + return self.platform_dir / "opentdf.yaml" @property def platform_template_config(self) -> Path: - """Platform config template path.""" - return self.platform_dir / "opentdf.yaml" + """Platform config template path (committed to git).""" + return self.platform_dir / "opentdf-dev.yaml" @property def kas_template_config(self) -> Path: diff --git a/otdf-local/src/otdf_local/services/docker.py b/otdf-local/src/otdf_local/services/docker.py index 911b42e3c..dd48483d2 100644 --- a/otdf-local/src/otdf_local/services/docker.py +++ b/otdf-local/src/otdf_local/services/docker.py @@ -1,7 +1,9 @@ """Docker compose service management.""" import json +import os import subprocess +from pathlib import Path from otdf_local.config.ports import Ports from otdf_local.config.settings import Settings @@ -12,9 +14,10 @@ class DockerService(Service): """Manages Docker compose services (Keycloak, PostgreSQL).""" - def __init__(self, settings: Settings) -> None: + def __init__(self, settings: Settings, keys_dir: Path | None = None) -> None: super().__init__(settings) self._compose_file = settings.docker_compose_file + self._keys_dir = keys_dir or settings.keys_dir @property def name(self) -> str: @@ -37,11 +40,13 @@ def start(self) -> bool: if not self._compose_file.exists(): return False + env = {**os.environ, "KEYS_DIR": str(self._keys_dir.resolve())} result = subprocess.run( ["docker", "compose", "-f", str(self._compose_file), "up", "-d"], capture_output=True, text=True, cwd=self._compose_file.parent, + env=env, ) return result.returncode == 0 @@ -140,10 +145,13 @@ def get_all_info(self) -> list[ServiceInfo]: ] -def get_docker_service(settings: Settings | None = None) -> DockerService: +def get_docker_service( + settings: Settings | None = None, + keys_dir: Path | None = None, +) -> DockerService: """Get a DockerService instance.""" if settings is None: from otdf_local.config.settings import get_settings settings = get_settings() - return DockerService(settings) + return DockerService(settings, keys_dir) diff --git a/otdf-local/src/otdf_local/services/platform.py b/otdf-local/src/otdf_local/services/platform.py index 15f7f4e5e..aba4e413d 100644 --- a/otdf-local/src/otdf_local/services/platform.py +++ b/otdf-local/src/otdf_local/services/platform.py @@ -12,6 +12,7 @@ kill_process_on_port, ) from otdf_local.services.base import Service, ServiceInfo, ServiceType +from otdf_local.config.overrides import apply_overrides, load_overrides from otdf_local.utils.keys import get_golden_keyring_entries, setup_golden_keys from otdf_local.utils.yaml import ( append_to_list, @@ -69,6 +70,13 @@ def _generate_config(self) -> Path: copy_yaml_with_updates(template_path, config_path, updates) + # Apply user-specified feature overrides + overrides = load_overrides(self.settings.xtest_root) + if overrides: + data = load_yaml(config_path) + apply_overrides(data, overrides) + save_yaml(config_path, data) + # Set up golden keys for legacy TDF tests self._setup_golden_keys(config_path) diff --git a/otdf-local/src/otdf_local/utils/console.py b/otdf-local/src/otdf_local/utils/console.py index cbb0e43f4..cb28e1876 100644 --- a/otdf-local/src/otdf_local/utils/console.py +++ b/otdf-local/src/otdf_local/utils/console.py @@ -8,8 +8,9 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from rich.table import Table -# Global console instance -console = Console() +# Global console instance (stderr for all status/diagnostic output) +console = Console(stderr=True) +_stdout_console = Console() # stdout for machine-readable data output def print_success(message: str) -> None: @@ -71,6 +72,11 @@ def format_health(healthy: bool | None) -> str: return "[red]✗[/red]" +def print_json(data: str) -> None: + """Print JSON data to stdout.""" + _stdout_console.print_json(data) + + def create_progress() -> Progress: """Create a progress display for multi-step operations.""" return Progress( diff --git a/otdf-local/src/otdf_local/utils/keys.py b/otdf-local/src/otdf_local/utils/keys.py index dee84f2af..606b9139c 100644 --- a/otdf-local/src/otdf_local/utils/keys.py +++ b/otdf-local/src/otdf_local/utils/keys.py @@ -1,10 +1,16 @@ """Cryptographic key generation utilities.""" import json +import logging +import os +import platform import secrets import subprocess +import tempfile from pathlib import Path +logger = logging.getLogger(__name__) + def generate_root_key() -> str: """Generate a random 256-bit root key as hex string.""" @@ -157,6 +163,323 @@ def ensure_keys_exist(key_dir: Path, force: bool = False) -> bool: return True +_KEYCLOAK_KEY_FILES = [ + "keycloak-ca.pem", + "keycloak-ca-private.pem", + "localhost.crt", + "localhost.key", + "sampleuser.crt", + "sampleuser.key", + "ca.p12", + "ca.jks", +] + + +def generate_ca_keypair(keys_dir: Path) -> tuple[Path, Path]: + """Generate a self-signed CA keypair for Keycloak TLS. + + Args: + keys_dir: Directory to store keys + + Returns: + Tuple of (ca_private_key_path, ca_cert_path) + """ + keys_dir.mkdir(parents=True, exist_ok=True) + ca_key = keys_dir / "keycloak-ca-private.pem" + ca_cert = keys_dir / "keycloak-ca.pem" + + subprocess.run( + [ + "openssl", "req", "-x509", "-nodes", + "-newkey", "RSA:2048", + "-subj", "/CN=ca", + "-keyout", str(ca_key), + "-out", str(ca_cert), + "-days", "365", + ], + check=True, + capture_output=True, + ) + ca_key.chmod(0o600) + + return ca_key, ca_cert + + +def generate_localhost_cert(keys_dir: Path) -> tuple[Path, Path]: + """Generate a localhost certificate signed by the CA. + + Creates a server cert with SAN DNS:localhost,IP:127.0.0.1 + suitable for Keycloak HTTPS. + + Args: + keys_dir: Directory containing CA keys and to store output + + Returns: + Tuple of (key_path, cert_path) + """ + ca_key = keys_dir / "keycloak-ca-private.pem" + ca_cert = keys_dir / "keycloak-ca.pem" + server_key = keys_dir / "localhost.key" + server_cert = keys_dir / "localhost.crt" + + with tempfile.TemporaryDirectory() as tmpdir: + tmp = Path(tmpdir) + csr_path = tmp / "localhost.req" + san_conf = tmp / "sanX509.conf" + req_conf = tmp / "req.conf" + + san_conf.write_text("subjectAltName=DNS:localhost,IP:127.0.0.1") + req_conf.write_text( + "[req]\n" + "distinguished_name=req_distinguished_name\n" + "[req_distinguished_name]\n" + "[alt_names]\n" + "DNS.1=localhost\n" + "IP.1=127.0.0.1\n" + ) + + # Generate CSR + key + subprocess.run( + [ + "openssl", "req", "-new", "-nodes", + "-newkey", "rsa:2048", + "-keyout", str(server_key), + "-out", str(csr_path), + "-batch", + "-subj", "/CN=localhost", + "-config", str(req_conf), + ], + check=True, + capture_output=True, + ) + server_key.chmod(0o600) + + # Sign with CA + subprocess.run( + [ + "openssl", "x509", "-req", + "-in", str(csr_path), + "-CA", str(ca_cert), + "-CAkey", str(ca_key), + "-CAcreateserial", + "-out", str(server_cert), + "-days", "3650", + "-sha256", + "-extfile", str(san_conf), + ], + check=True, + capture_output=True, + ) + + # Clean up CA serial file if created in keys_dir + serial_file = keys_dir / "keycloak-ca.srl" + if serial_file.exists(): + serial_file.unlink() + + return server_key, server_cert + + +def generate_sampleuser_cert(keys_dir: Path) -> tuple[Path, Path]: + """Generate a sample user client certificate signed by the CA. + + Args: + keys_dir: Directory containing CA keys and to store output + + Returns: + Tuple of (key_path, cert_path) + """ + ca_key = keys_dir / "keycloak-ca-private.pem" + ca_cert = keys_dir / "keycloak-ca.pem" + user_key = keys_dir / "sampleuser.key" + user_cert = keys_dir / "sampleuser.crt" + + with tempfile.TemporaryDirectory() as tmpdir: + csr_path = Path(tmpdir) / "sampleuser.req" + + subprocess.run( + [ + "openssl", "req", "-new", "-nodes", + "-newkey", "rsa:2048", + "-keyout", str(user_key), + "-out", str(csr_path), + "-batch", + "-subj", "/CN=sampleuser", + ], + check=True, + capture_output=True, + ) + user_key.chmod(0o600) + + subprocess.run( + [ + "openssl", "x509", "-req", + "-in", str(csr_path), + "-CA", str(ca_cert), + "-CAkey", str(ca_key), + "-CAcreateserial", + "-out", str(user_cert), + "-days", "3650", + ], + check=True, + capture_output=True, + ) + + serial_file = keys_dir / "keycloak-ca.srl" + if serial_file.exists(): + serial_file.unlink() + + return user_key, user_cert + + +def _get_java_opts_for_docker() -> list[str]: + """Get JAVA_TOOL_OPTIONS env args for Docker keytool. + + Works around SIGILL on Apple M4 chips by disabling SVE. + """ + if platform.machine() != "arm64": + return [] + try: + result = subprocess.run( + ["sysctl", "-n", "machdep.cpu.brand_string"], + capture_output=True, text=True, timeout=5, + ) + if result.returncode == 0 and "M4" in result.stdout: + return ["-e", "JAVA_TOOL_OPTIONS=-XX:UseSVE=0"] + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + return [] + + +def _get_keycloak_image(compose_file: Path | None) -> str: + """Extract the Keycloak image from a docker-compose file, or use default.""" + default = "keycloak/keycloak:25.0" + if compose_file is None or not compose_file.exists(): + return default + try: + text = compose_file.read_text() + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("image:") and "keycloak" in stripped: + return stripped.split(":", 1)[1].strip() + except OSError: + pass + return default + + +def generate_ca_stores( + keys_dir: Path, + compose_file: Path | None = None, +) -> tuple[Path, Path]: + """Generate PKCS12 and JKS keystores from the CA cert. + + Args: + keys_dir: Directory containing CA keys + compose_file: Optional docker-compose.yaml to extract keycloak image version + + Returns: + Tuple of (p12_path, jks_path) + """ + ca_key = keys_dir / "keycloak-ca-private.pem" + ca_cert = keys_dir / "keycloak-ca.pem" + p12_path = keys_dir / "ca.p12" + jks_path = keys_dir / "ca.jks" + + # Generate PKCS12 + subprocess.run( + [ + "openssl", "pkcs12", "-export", + "-in", str(ca_cert), + "-inkey", str(ca_key), + "-out", str(p12_path), + "-nodes", + "-passout", "pass:password", + ], + check=True, + capture_output=True, + ) + + # Generate JKS via Docker keytool + keycloak_image = _get_keycloak_image(compose_file) + java_opts = _get_java_opts_for_docker() + uid_gid = f"{os.getuid()}:{os.getgid()}" + + cmd = [ + "docker", "run", "--rm", + *java_opts, + "-v", f"{keys_dir.resolve()}:/keys", + "--entrypoint", "keytool", + "--user", uid_gid, + keycloak_image, + "-importkeystore", + "-srckeystore", "/keys/ca.p12", + "-srcstoretype", "PKCS12", + "-destkeystore", "/keys/ca.jks", + "-deststoretype", "JKS", + "-srcstorepass", "password", + "-deststorepass", "password", + "-noprompt", + ] + + subprocess.run(cmd, check=True, capture_output=True) + + return p12_path, jks_path + + +def ensure_keycloak_keys( + keys_dir: Path, + compose_file: Path | None = None, + force: bool = False, +) -> bool: + """Ensure all Keycloak TLS keys exist, generating if needed. + + All keycloak keys are regenerated together since the certs form a + CA chain (localhost.crt and sampleuser.crt are signed by the CA). + + Args: + keys_dir: Directory for key storage + compose_file: Optional docker-compose.yaml for keycloak image version + force: If True, regenerate all keys even if they exist + + Returns: + True if keys were generated, False if they already existed + """ + if not force and all((keys_dir / f).exists() for f in _KEYCLOAK_KEY_FILES): + return False + + logger.info("Generating Keycloak TLS keys in %s", keys_dir) + generate_ca_keypair(keys_dir) + generate_localhost_cert(keys_dir) + generate_sampleuser_cert(keys_dir) + generate_ca_stores(keys_dir, compose_file) + return True + + +def ensure_all_temp_keys( + platform_dir: Path, + keys_dir: Path, + compose_file: Path | None = None, + force: bool = False, +) -> bool: + """Ensure all temporary keys exist for running the platform. + + KAS keys are generated per-platform-dir (referenced by relative paths + in the platform config). Keycloak TLS keys are generated in a shared + keys_dir so they can be reused across platform variants. + + Args: + platform_dir: Platform source directory (for KAS keys) + keys_dir: Shared directory for Keycloak TLS keys + compose_file: Optional docker-compose.yaml for keycloak image version + force: If True, regenerate all keys + + Returns: + True if any keys were generated + """ + kas_generated = ensure_keys_exist(platform_dir, force) + kc_generated = ensure_keycloak_keys(keys_dir, compose_file, force) + return kas_generated or kc_generated + + def setup_golden_keys( xtest_root: Path, platform_dir: Path, diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py index e62ae2464..c29307598 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/cli_install.py @@ -87,3 +87,31 @@ def artifact( except InstallError as e: typer.echo(f"Error: {e}", err=True) raise typer.Exit(1) + + +@install_app.command() +def variant( + name: Annotated[str, typer.Argument(help="Variant name (e.g., gemini, enhanced, codex)")], + platform_dir: Annotated[ + str, typer.Argument(help="Path to the platform variant directory") + ], + branch: Annotated[ + str, typer.Option(help="otdfctl branch to checkout for source") + ] = "main", +) -> None: + """Build Go SDK (otdfctl) linked against a platform variant's modules. + + Generates a variant-specific go.work file and builds otdfctl using the + platform variant's lib/ocrypto, sdk, and protocol modules. This allows + testing different post-quantum implementations side by side. + + Example: + otdf-sdk-mgr install variant gemini ~/repos/pq-gemini/platform + """ + from otdf_sdk_mgr.installers import InstallError, cmd_variant + + try: + cmd_variant(name, platform_dir, branch) + except InstallError as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) diff --git a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py index 6136d4015..7016a6c15 100644 --- a/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py +++ b/otdf-sdk-mgr/src/otdf_sdk_mgr/installers.py @@ -258,3 +258,104 @@ def cmd_install( """Install a single SDK version (used by CI action).""" print(f"Installing {sdk} {version}...") install_release(sdk, version, dist_name=dist_name, source=source) + + +# Modules that otdfctl depends on from the platform monorepo. +# Maps module path suffix to directory relative to platform root. +_PLATFORM_MODULES = { + "lib/ocrypto": "lib/ocrypto", + "lib/flattening": "lib/flattening", + "lib/identifier": "lib/identifier", + "protocol/go": "protocol/go", + "sdk": "sdk", +} + + +def _generate_gowork(otdfctl_src: Path, variant_name: str, platform_dir: Path) -> Path: + """Generate a go.work file for building otdfctl against a platform variant. + + Args: + otdfctl_src: Path to the otdfctl source checkout (e.g., sdk/go/src/main/) + variant_name: Name for this variant (e.g., "gemini") + platform_dir: Absolute path to the platform variant directory + + Returns: + Path to the generated go.work file + """ + platform_dir = platform_dir.resolve() + + # Read Go version from the platform's go.work if available + go_version = "1.25.0" + toolchain = "go1.25.8" + platform_gowork = platform_dir / "go.work" + if platform_gowork.exists(): + for line in platform_gowork.read_text().splitlines(): + line = line.strip() + if line.startswith("go ") and not line.startswith("go."): + go_version = line.split()[1] + elif line.startswith("toolchain "): + toolchain = line.split()[1] + + # Validate that required platform modules exist + use_dirs = [] + for _mod_suffix, rel_dir in _PLATFORM_MODULES.items(): + mod_dir = platform_dir / rel_dir + if not (mod_dir / "go.mod").exists(): + raise InstallError( + f"Platform module directory {mod_dir} does not contain go.mod. " + f"Ensure {platform_dir} is a valid platform checkout." + ) + use_dirs.append(str(mod_dir)) + + gowork_path = otdfctl_src / f"go.work.{variant_name}" + lines = [ + f"go {go_version}", + "", + f"toolchain {toolchain}", + "", + "use .", + "", + "use (", + ] + for d in use_dirs: + lines.append(f"\t{d}") + lines.append(")") + lines.append("") + + gowork_path.write_text("\n".join(lines)) + print(f" Generated {gowork_path}") + return gowork_path + + +def cmd_variant(name: str, platform_dir: str, branch: str = "main") -> None: + """Build otdfctl linked against a specific platform variant's modules. + + 1. Ensures otdfctl source is checked out + 2. Generates a variant-specific go.work file + 3. Builds via the Makefile build-variant target + """ + platform_path = Path(platform_dir).resolve() + if not platform_path.is_dir(): + raise InstallError(f"Platform directory does not exist: {platform_path}") + + sdk_dirs = get_sdk_dirs() + go_dir = sdk_dirs["go"] + src_dir = go_dir / "src" / branch.replace("/", "--") + + # Ensure otdfctl source is checked out + if not src_dir.exists(): + print(f"Checking out otdfctl from branch '{branch}'...") + checkout_sdk_branch("go", branch) + else: + print(f"Using existing otdfctl checkout at {src_dir}") + + # Generate variant-specific go.work + _generate_gowork(src_dir, name, platform_path) + + # Build using the Makefile build-variant target + print(f"Building variant '{name}' against {platform_path}...") + subprocess.check_call( + ["make", "build-variant", f"VARIANT={name}"], + cwd=go_dir, + ) + print(f" Variant '{name}' built to {go_dir / 'dist' / name}") diff --git a/xtest/conftest.py b/xtest/conftest.py index ce6a71317..dee1a8ceb 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -60,6 +60,20 @@ def is_a(v: str) -> typing.Any: return is_a +def is_sdk_spec_list(t: typing.Any) -> typing.Callable[[str], typing.Any]: + """Validate SDK specs: bare names ('go') or qualified ('go:gemini').""" + valid_sdks = typing.get_args(t) + + def is_a(v: str) -> typing.Any: + for i in v.split(): + sdk_name = i.split(":")[0] if ":" in i else i + if sdk_name not in valid_sdks: + raise ValueError(f"Invalid SDK '{sdk_name}' in '{i}', must be one of {valid_sdks}") + return v + + return is_a + + def pytest_addoption(parser: pytest.Parser): """Add custom CLI options for pytest.""" parser.addoption( @@ -94,18 +108,21 @@ def pytest_addoption(parser: pytest.Parser): ) parser.addoption( "--sdks", - help=f"select which sdks to run by default, unless overridden, one or more of {englist(typing.get_args(tdfs.sdk_type))}", - type=is_type_or_list_of_types(tdfs.sdk_type), + help=( + f"select which sdks to run, one or more of {englist(typing.get_args(tdfs.sdk_type))}. " + "Use sdk:version to select a specific version (e.g., 'go:gemini')" + ), + type=is_sdk_spec_list(tdfs.sdk_type), ) parser.addoption( "--sdks-decrypt", - help="select which sdks to run for decrypt only", - type=is_type_or_list_of_types(tdfs.sdk_type), + help="select which sdks for decrypt (supports sdk:version syntax)", + type=is_sdk_spec_list(tdfs.sdk_type), ) parser.addoption( "--sdks-encrypt", - help="select which sdks to run for encrypt only", - type=is_type_or_list_of_types(tdfs.sdk_type), + help="select which sdks for encrypt (supports sdk:version syntax)", + type=is_sdk_spec_list(tdfs.sdk_type), ) @@ -139,44 +156,31 @@ def list_opt(name: str, t: typing.Any) -> list[str]: raise ValueError(f"Invalid value for {name}: {i}, must be one of {ttt}") return a - def defaulted_list_opt[T]( - names: list[str], t: typing.Any, default: list[T] - ) -> list[T]: + def sdk_specs_from_opts(names: list[str]) -> list[str] | None: + """Get raw SDK spec strings from the first non-empty option.""" for name in names: v = metafunc.config.getoption(name) - if v: - return cast(list[T], list_opt(name, t)) - return default + if v and isinstance(v, str): + return v.split() + return None + + def resolve_sdk_specs(specs: list[str] | None) -> list[tdfs.SDK]: + """Resolve SDK specs (bare or qualified) into SDK objects.""" + if specs is None: + # Default: all versions of all SDKs + specs = list(typing.get_args(tdfs.sdk_type)) + return [sdk for spec in specs for sdk in tdfs.parse_sdk_spec(spec)] subject_sdks: set[tdfs.SDK] = set() if "encrypt_sdk" in metafunc.fixturenames: - encrypt_sdks: list[tdfs.sdk_type] = [] - encrypt_sdks = defaulted_list_opt( - ["--sdks-encrypt", "--sdks"], - tdfs.sdk_type, - list(typing.get_args(tdfs.sdk_type)), - ) - # convert list of sdk_type to list of SDK objects - e_sdks = [ - v - for sdks in [tdfs.all_versions_of(sdk) for sdk in encrypt_sdks] - for v in sdks - ] + e_specs = sdk_specs_from_opts(["--sdks-encrypt", "--sdks"]) + e_sdks = resolve_sdk_specs(e_specs) metafunc.parametrize("encrypt_sdk", e_sdks, ids=[str(x) for x in e_sdks]) subject_sdks |= set(e_sdks) if "decrypt_sdk" in metafunc.fixturenames: - decrypt_sdks: list[tdfs.sdk_type] = [] - decrypt_sdks = defaulted_list_opt( - ["--sdks-decrypt", "--sdks"], - tdfs.sdk_type, - list(typing.get_args(tdfs.sdk_type)), - ) - d_sdks = [ - v - for sdks in [tdfs.all_versions_of(sdk) for sdk in decrypt_sdks] - for v in sdks - ] + d_specs = sdk_specs_from_opts(["--sdks-decrypt", "--sdks"]) + d_sdks = resolve_sdk_specs(d_specs) metafunc.parametrize("decrypt_sdk", d_sdks, ids=[str(x) for x in d_sdks]) subject_sdks |= set(d_sdks) diff --git a/xtest/run-pqc-matrix.sh b/xtest/run-pqc-matrix.sh new file mode 100755 index 000000000..b38851b8e --- /dev/null +++ b/xtest/run-pqc-matrix.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Run the PQC X-Wing test matrix: SDK variant x backend variant. +# +# Prerequisites: +# 1. otdf-sdk-mgr and otdf-local installed (uv tool install --editable) +# 2. Docker running (for Keycloak + Postgres) +# +# Usage: +# ./run-pqc-matrix.sh # Full 3x3 matrix +# ./run-pqc-matrix.sh --build # Build SDK variants only (no tests) +# ./run-pqc-matrix.sh --diagonal # Only test matching SDK/backend pairs +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +# --- Variant definitions (override with env vars) --- +VARIANTS=(gemini enhanced codex) +PLATFORM_DIRS=( + "${PQC_GEMINI_DIR:-$HOME/Documents/GitHub/post-quantum-hybrid-gemini-2026-03-dm/platform}" + "${PQC_ENHANCED_DIR:-$HOME/Documents/GitHub/post-quantum-enhanced-2026-03-dm/platform}" + "${PQC_CODEX_DIR:-$HOME/Documents/GitHub/post-quantum-hybrid-codex-2026-03-dm/platform}" +) + +# --- Options --- +BUILD_ONLY=false +DIAGONAL_ONLY=false +PYTEST_ARGS=() + +for arg in "$@"; do + case "$arg" in + --build) BUILD_ONLY=true ;; + --diagonal) DIAGONAL_ONLY=true ;; + *) PYTEST_ARGS+=("$arg") ;; + esac +done + +# --- Phase 1: Build all SDK variants --- +echo "=== Phase 1: Building SDK variants ===" +for i in "${!VARIANTS[@]}"; do + variant="${VARIANTS[$i]}" + platform_dir="${PLATFORM_DIRS[$i]}" + if [ ! -d "$platform_dir" ]; then + echo "WARNING: Platform dir not found: $platform_dir (skipping $variant)" + continue + fi + echo "--- Building $variant from $platform_dir ---" + otdf-sdk-mgr install variant "$variant" "$platform_dir" +done + +if $BUILD_ONLY; then + echo "=== Build complete. SDK variants in sdk/go/dist/ ===" + ls -la sdk/go/dist/ + exit 0 +fi + +# --- Phase 2: Run test matrix --- +echo "" +echo "=== Phase 2: Running test matrix ===" +mkdir -p results + +OTDF_LOCAL_DIR="$SCRIPT_DIR/../otdf-local" +PASS=0 +FAIL=0 +SKIP=0 +SUMMARY="" + +for bi in "${!VARIANTS[@]}"; do + backend="${VARIANTS[$bi]}" + backend_dir="${PLATFORM_DIRS[$bi]}" + + if [ ! -d "$backend_dir" ]; then + echo "WARNING: Backend dir not found: $backend_dir (skipping)" + SKIP=$((SKIP + 1)) + continue + fi + + echo "" + echo "=== Starting backend: $backend ===" + export OTDF_LOCAL_PLATFORM_DIR="$backend_dir" + + (cd "$OTDF_LOCAL_DIR" && uv run otdf-local down 2>/dev/null) || true + if ! (cd "$OTDF_LOCAL_DIR" && uv run otdf-local up); then + echo "ERROR: Failed to start backend $backend" + SUMMARY+=" BACKEND-FAIL $backend"$'\n' + SKIP=$((SKIP + ${#VARIANTS[@]})) + continue + fi + + for si in "${!VARIANTS[@]}"; do + sdk="${VARIANTS[$si]}" + + # Diagonal mode: skip non-matching pairs + if $DIAGONAL_ONLY && [ "$si" != "$bi" ]; then + continue + fi + + log_file="results/${sdk}-on-${backend}.log" + echo "" + echo "--- Testing SDK=$sdk Backend=$backend ---" + + set -a && source test.env && set +a + + if uv run pytest test_tdfs.py -k xwing \ + --sdks "go:$sdk" \ + -v --tb=short \ + "${PYTEST_ARGS[@]}" \ + 2>&1 | tee "$log_file"; then + result="PASS" + PASS=$((PASS + 1)) + else + result="FAIL" + FAIL=$((FAIL + 1)) + fi + SUMMARY+=" $result SDK=$sdk Backend=$backend"$'\n' + done +done + +# Shut down services +(cd "$OTDF_LOCAL_DIR" && uv run otdf-local down 2>/dev/null) || true + +echo "" +echo "============================================" +echo " PQC Matrix Results" +echo "============================================" +echo "$SUMMARY" +echo " Total: $((PASS + FAIL + SKIP)) Pass: $PASS Fail: $FAIL Skip: $SKIP" +echo "" +echo " Logs in: $SCRIPT_DIR/results/" +echo "============================================" diff --git a/xtest/sdk/go/Makefile b/xtest/sdk/go/Makefile index 12b869dc5..d172b1360 100644 --- a/xtest/sdk/go/Makefile +++ b/xtest/sdk/go/Makefile @@ -46,6 +46,26 @@ build: done @echo "All binaries built successfully" +# Build a variant using a specific go.work file from src/main/ +# Usage: make build-variant VARIANT=gemini +# Requires: src/main/go.work.$(VARIANT) must exist +build-variant: + @test -n "$(VARIANT)" || { echo "Error: VARIANT is required (e.g., make build-variant VARIANT=gemini)"; exit 1; } + @test -f "$(MAKEFILE_DIR)/src/main/go.work.$(VARIANT)" || { echo "Error: $(MAKEFILE_DIR)/src/main/go.work.$(VARIANT) not found"; exit 1; } + @echo "Building variant $(VARIANT) with GOWORK=go.work.$(VARIANT)" + cd $(MAKEFILE_DIR)/src/main && \ + GOWORK=$(abspath $(MAKEFILE_DIR)/src/main/go.work.$(VARIANT)) \ + go build -o $(MAKEFILE_DIR)/binary-$(VARIANT) . || { \ + echo "Error: Go build failed for variant $(VARIANT)"; \ + exit 1; \ + } + mkdir -p $(MAKEFILE_DIR)/dist/$(VARIANT) + mv $(MAKEFILE_DIR)/binary-$(VARIANT) $(MAKEFILE_DIR)/dist/$(VARIANT)/otdfctl + cp $(MAKEFILE_DIR)/cli.sh $(MAKEFILE_DIR)/dist/$(VARIANT)/ + cp $(MAKEFILE_DIR)/otdfctl.sh $(MAKEFILE_DIR)/dist/$(VARIANT)/ + cp $(MAKEFILE_DIR)/opentdfctl.yaml $(MAKEFILE_DIR)/dist/$(VARIANT)/ + @echo "Variant $(VARIANT) built successfully" + clean: @echo "Cleaning up binaries" @for version in $(VERSIONS); do \ diff --git a/xtest/tdfs.py b/xtest/tdfs.py index 0c4c9c611..87fc06dca 100644 --- a/xtest/tdfs.py +++ b/xtest/tdfs.py @@ -515,6 +515,18 @@ def all_versions_of(sdk: sdk_type) -> list[SDK]: return versions +def parse_sdk_spec(spec: str) -> list[SDK]: + """Parse an SDK spec like 'go' or 'go:gemini' into a list of SDK objects. + + Bare names (e.g., 'go') return all discovered versions. + Qualified names (e.g., 'go:gemini') return a single specific version. + """ + if ":" in spec: + sdk_name, version = spec.split(":", 1) + return [SDK(sdk_name, version)] + return all_versions_of(spec) + + def skip_if_unsupported(sdk: SDK, *features: feature_type): pfs = get_platform_features() pfs.skip_if_unsupported(*features)