diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index 484c2593..c7f2ceda 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -190,6 +190,8 @@ jobs: data_agent/test_metadata_fabric_lineage_delivery.py \ data_agent/test_metadata_fabric_lineage_delivery_postgres.py \ data_agent/test_metadata_fabric_provider_identity.py \ + data_agent/test_metadata_fabric_gravitino_identity.py \ + data_agent/test_metadata_fabric_identity_gate.py \ data_agent/test_metadata_fabric_otel_failure_rehearsal.py \ data_agent/test_metadata_fabric_otel_metrics.py \ data_agent/test_metadata_fabric_provider_metrics.py \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e39a739d..ee3b037c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,12 @@ jobs: - name: Validate metadata fabric provider identity evidence run: python -m data_agent.metadata_fabric_provider_identity validate + - name: Validate metadata fabric Gravitino identity evidence + run: python -m data_agent.metadata_fabric_gravitino_identity validate + + - name: Validate metadata fabric production identity gate + run: python -m data_agent.metadata_fabric_identity_gate validate + - name: Validate DolphinScheduler adapter boundary run: python -m data_agent.dolphinscheduler_adapter validate @@ -176,6 +182,8 @@ jobs: data_agent/test_metadata_fabric_binding_contract.py \ data_agent/test_metadata_fabric_lineage_delivery.py \ data_agent/test_metadata_fabric_provider_identity.py \ + data_agent/test_metadata_fabric_gravitino_identity.py \ + data_agent/test_metadata_fabric_identity_gate.py \ data_agent/test_metadata_fabric_otel_failure_rehearsal.py \ data_agent/test_metadata_fabric_otel_metrics.py \ data_agent/test_metadata_fabric_provider_metrics.py \ diff --git a/config/metadata-fabric-gravitino-identity.local.yaml b/config/metadata-fabric-gravitino-identity.local.yaml new file mode 100644 index 00000000..d2d5a6d5 --- /dev/null +++ b/config/metadata-fabric-gravitino-identity.local.yaml @@ -0,0 +1,64 @@ +schema: gda.metadata_fabric_gravitino_identity_profile.v1 +environment: local_docker_desktop + +cluster: + context: docker-desktop + source_namespace: gda-metadata-sandbox + rehearsal_namespace: gda-metadata-identity + source_schema_configmap: metadata-gravitino-schema-1-3-0 + +runtime: + manifest: k8s/metadata-fabric-gravitino-identity + gravitino_version: 1.3.0 + gravitino_image: gda/gravitino:1.3.0-local-arm64 + postgresql_version: 16.10-bookworm + postgresql_image: postgres:16.10-bookworm + service: gravitino-identity + service_port: 8090 + authenticator: basic + idp_extension: org.apache.gravitino.idp.web.rest.feature + access_control_enabled: true + service_account: gravitino-identity + service_account_automount_disabled: true + transport: local_loopback_http + +identity: + service_admin: gda-identity-admin + user: gda-metadata-projection + role: gda-table-projection + material_delivery: runtime_generated_ephemeral_kubernetes_object + login_rotation: administrator_reset + revocation: idp_user_delete + +scope: + metalake: gda_identity + catalog: lakehouse + schema: published + table: gda_identity_probe + denied_catalog: unauthorized_catalog + role_securable_objects: + - full_name: lakehouse + type: CATALOG + privileges: + - name: USE_CATALOG + condition: ALLOW + - full_name: lakehouse.published + type: SCHEMA + privileges: + - name: CREATE_TABLE + condition: ALLOW + - name: USE_SCHEMA + condition: ALLOW + +claims: + local_gravitino_basic_identity_verified: false + local_gravitino_minimum_privilege_verified: false + local_gravitino_login_rotation_verified: false + local_gravitino_revocation_verified: false + gravitino_authentication_verified: false + provider_minimum_privilege_verified: false + protected_workload_identity_verified: false + oidc_verified: false + tls_verified: false + production_identity_verified: false + production_ready: false diff --git a/config/metadata-fabric-identity.production.yaml b/config/metadata-fabric-identity.production.yaml new file mode 100644 index 00000000..9177b25b --- /dev/null +++ b/config/metadata-fabric-identity.production.yaml @@ -0,0 +1,132 @@ +schema: gda.metadata_fabric_identity_production_profile.v1 +environment: production + +scope: + providers: + openmetadata: + version: 1.13.1 + gravitino: + version: 1.3.0 + local_evidence: + openmetadata: + path: docs/evidence/metadata-fabric-provider-identity-2026-07-28.json + evidence_fingerprint: 61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f + gravitino: + path: docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json + evidence_fingerprint: f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd + +federation: + decision_status: pending + issuer: null + discovery_uri: null + jwks_uri: null + audience: null + token_exchange_mode: null + trust_policy_reference: null + workload_subject_claim: sub + tenant_claim: gda_tenant_id + maximum_token_ttl_seconds: 900 + +providers: + openmetadata: + integration_mode: null + authentication_component_reference: null + environment_binding: null + workload_identity_reference: null + kubernetes_service_account: null + namespace_template: null + direct_access_policy: deny_except_attested_identity_path + credential_delivery: short_lived_token_exchange + static_credentials_forbidden: true + rotation_mode: automatic_before_expiry + revocation_mode: idp_subject_disable + gravitino: + integration_mode: null + authentication_component_reference: null + environment_binding: null + workload_identity_reference: null + kubernetes_service_account: null + namespace_template: null + direct_access_policy: deny_except_attested_identity_path + credential_delivery: short_lived_token_exchange + static_credentials_forbidden: true + rotation_mode: automatic_before_expiry + revocation_mode: idp_subject_disable + +authorization: + openmetadata: + mandatory_roles: + - DefaultBotRole + project_role: GdaMetadataTableProjectionRole + project_permissions: + - resource: table + operations: + - Create + denied_probe: + resource: policy + operation: Create + gravitino: + role: gda-table-projection + securable_objects: + - full_name: lakehouse + type: CATALOG + privileges: + - USE_CATALOG + - full_name: lakehouse.published + type: SCHEMA + privileges: + - CREATE_TABLE + - USE_SCHEMA + denied_probe: + resource: METALAKE + operation: CREATE_CATALOG + +tls: + required: true + minimum_version: TLSv1.2 + openmetadata_endpoint: null + gravitino_endpoint: null + trust_bundle_reference: null + certificate_policy_reference: null + mtls_for_internal_hops: true + +catalog: + decision_status: pending + gravitino_backend: null + catalog_reference: null + persistence_reference: null + backup_policy_reference: null + persistent_required: true + +tenancy: + isolation_mode: null + policy_reference: null + tenant_claim: gda_tenant_id + cross_tenant_denial_required: true + +operations: + identity_owner: null + security_owner: null + incident_owner: null + audit_log_reference: null + rotation_slo_minutes: null + revocation_slo_minutes: null + runbook: + uri: null + version: null + rollback_runbook: + uri: null + version: null + +claims: + identity_decision_frozen: false + provider_minimum_privilege_verified: false + protected_workload_identity_verified: false + oidc_verified: false + tls_verified: false + credential_rotation_verified: false + credential_revocation_verified: false + persistent_catalog_identity_binding_verified: false + production_identity_verified: false + production_identity_gate_passed: false + production_ready: false diff --git a/data_agent/metadata_fabric_gravitino_identity.py b/data_agent/metadata_fabric_gravitino_identity.py new file mode 100644 index 00000000..6c9de899 --- /dev/null +++ b/data_agent/metadata_fabric_gravitino_identity.py @@ -0,0 +1,1414 @@ +"""Rehearse authenticated, bounded Gravitino access in an isolated namespace. + +The rehearsal starts a temporary Gravitino 1.3.0 server backed by PostgreSQL, +enables the built-in Basic IdP and deny-by-default authorization, and grants one +named user only the privileges needed to create a table in one schema. It then +proves an allowed table mutation, a denied catalog mutation, login rotation, +revocation, and complete namespace cleanup. This remains local POC evidence, +not protected OIDC or production identity. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import secrets +import subprocess +import sys +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Annotated, Any, Literal +from urllib.parse import quote +from uuid import UUID + +import httpx +import yaml +from pydantic import BaseModel, ConfigDict, Field, SecretStr, StringConstraints + +from . import metadata_fabric_ingestion_replay as ingestion_replay +from . import metadata_fabric_provider_metrics as provider_metrics +from . import metadata_fabric_recovery_rehearsal as recovery + + +PROFILE_SCHEMA = "gda.metadata_fabric_gravitino_identity_profile.v1" +CONTRACT_SCHEMA = "gda.metadata_fabric_gravitino_identity_contract.v1" +OBSERVATION_SCHEMA = "gda.metadata_fabric_gravitino_identity_observation.v1" +EVIDENCE_SCHEMA = "gda.metadata_fabric_gravitino_identity_evidence.v1" +VALIDATION_SCHEMA = "gda.metadata_fabric_gravitino_identity_validation.v1" + +CONTEXT = "docker-desktop" +SOURCE_NAMESPACE = "gda-metadata-sandbox" +REHEARSAL_NAMESPACE = "gda-metadata-identity" +GRAVITINO_SCHEMA_SHA256 = ( + "7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df" +) + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILE_PATH = ( + REPO_ROOT / "config/metadata-fabric-gravitino-identity.local.yaml" +) +DEFAULT_EVIDENCE_PATH = ( + REPO_ROOT / "docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json" +) +DEFAULT_WRAPPER_PATH = REPO_ROOT / "scripts/metadata-fabric-gravitino-identity.sh" +MANIFEST_DIR = REPO_ROOT / "k8s/metadata-fabric-gravitino-identity" + +NonEmptyText = Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=1024), +] + + +class MetadataFabricGravitinoIdentityError(RuntimeError): + """The Gravitino identity contract or live rehearsal failed closed.""" + + +class ProviderRequestError(MetadataFabricGravitinoIdentityError): + """A Gravitino request failed without exposing response or login material.""" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class ClusterProfile(_FrozenModel): + context: Literal["docker-desktop"] + source_namespace: Literal["gda-metadata-sandbox"] + rehearsal_namespace: Literal["gda-metadata-identity"] + source_schema_configmap: Literal["metadata-gravitino-schema-1-3-0"] + + +class RuntimeProfile(_FrozenModel): + manifest: Literal["k8s/metadata-fabric-gravitino-identity"] + gravitino_version: Literal["1.3.0"] + gravitino_image: Literal["gda/gravitino:1.3.0-local-arm64"] + postgresql_version: Literal["16.10-bookworm"] + postgresql_image: Literal["postgres:16.10-bookworm"] + service: Literal["gravitino-identity"] + service_port: Literal[8090] + authenticator: Literal["basic"] + idp_extension: Literal["org.apache.gravitino.idp.web.rest.feature"] + access_control_enabled: Literal[True] + service_account: Literal["gravitino-identity"] + service_account_automount_disabled: Literal[True] + transport: Literal["local_loopback_http"] + + +class IdentityProfile(_FrozenModel): + service_admin: Literal["gda-identity-admin"] + user: Literal["gda-metadata-projection"] + role: Literal["gda-table-projection"] + material_delivery: Literal["runtime_generated_ephemeral_kubernetes_object"] + login_rotation: Literal["administrator_reset"] + revocation: Literal["idp_user_delete"] + + +class PrivilegeProfile(_FrozenModel): + name: Literal["USE_CATALOG", "USE_SCHEMA", "CREATE_TABLE"] + condition: Literal["ALLOW"] + + +class SecurableObjectProfile(_FrozenModel): + full_name: Literal["lakehouse", "lakehouse.published"] + type: Literal["CATALOG", "SCHEMA"] + privileges: tuple[PrivilegeProfile, ...] + + +class ScopeProfile(_FrozenModel): + metalake: Literal["gda_identity"] + catalog: Literal["lakehouse"] + schema_name: Literal["published"] = Field(alias="schema") + table: Literal["gda_identity_probe"] + denied_catalog: Literal["unauthorized_catalog"] + role_securable_objects: tuple[SecurableObjectProfile, ...] + + +class ClaimProfile(_FrozenModel): + local_gravitino_basic_identity_verified: Literal[False] + local_gravitino_minimum_privilege_verified: Literal[False] + local_gravitino_login_rotation_verified: Literal[False] + local_gravitino_revocation_verified: Literal[False] + gravitino_authentication_verified: Literal[False] + provider_minimum_privilege_verified: Literal[False] + protected_workload_identity_verified: Literal[False] + oidc_verified: Literal[False] + tls_verified: Literal[False] + production_identity_verified: Literal[False] + production_ready: Literal[False] + + +class GravitinoIdentityProfile(_FrozenModel): + schema_name: Literal["gda.metadata_fabric_gravitino_identity_profile.v1"] = ( + Field(alias="schema") + ) + environment: Literal["local_docker_desktop"] + cluster: ClusterProfile + runtime: RuntimeProfile + identity: IdentityProfile + scope: ScopeProfile + claims: ClaimProfile + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _valid_uuid(value: Any) -> bool: + try: + UUID(str(value)) + except (TypeError, ValueError): + return False + return True + + +def _valid_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _expected_securable_objects() -> list[dict[str, Any]]: + return [ + { + "fullName": "lakehouse", + "type": "CATALOG", + "privileges": [{"name": "USE_CATALOG", "condition": "ALLOW"}], + }, + { + "fullName": "lakehouse.published", + "type": "SCHEMA", + "privileges": [ + {"name": "CREATE_TABLE", "condition": "ALLOW"}, + {"name": "USE_SCHEMA", "condition": "ALLOW"}, + ], + }, + ] + + +def _profile_securable_objects( + profile: GravitinoIdentityProfile, +) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for item in profile.scope.role_securable_objects: + result.append( + { + "fullName": item.full_name, + "type": item.type, + "privileges": sorted( + [entry.model_dump(mode="json") for entry in item.privileges], + key=lambda entry: entry["name"], + ), + } + ) + return sorted(result, key=lambda item: item["fullName"]) + + +def load_profile(path: Path = DEFAULT_PROFILE_PATH) -> GravitinoIdentityProfile: + try: + value = yaml.safe_load(path.resolve().read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("Gravitino identity profile must be an object") + ingestion_replay._reject_sensitive_fields(value) + profile = GravitinoIdentityProfile.model_validate(value) + except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity profile is invalid" + ) from exc + if _profile_securable_objects(profile) != _expected_securable_objects(): + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity role exceeds the bounded table-create scope" + ) + return profile + + +def _manifest_documents() -> list[dict[str, Any]]: + documents: list[dict[str, Any]] = [] + for path in sorted(MANIFEST_DIR.glob("*.yaml")): + if path.name == "kustomization.yaml": + continue + for value in yaml.safe_load_all(path.read_text(encoding="utf-8")): + if isinstance(value, dict): + documents.append(value) + return documents + + +def _validate_manifest() -> list[str]: + errors: list[str] = [] + try: + documents = _manifest_documents() + except (OSError, yaml.YAMLError) as exc: + return [f"Gravitino identity manifest is invalid: {type(exc).__name__}"] + if any(document.get("kind") == "Secret" for document in documents): + errors.append("Gravitino identity manifest may not commit Secret values") + kinds = {str(document.get("kind")) for document in documents} + if not {"Namespace", "ServiceAccount", "ConfigMap", "Service", "StatefulSet"}.issubset( + kinds + ): + errors.append("Gravitino identity manifest is incomplete") + rendered = json.dumps(documents, ensure_ascii=True, sort_keys=True) + for marker in ( + "gravitino.authenticators = basic", + "gravitino.authorization.enable = true", + "org.apache.gravitino.idp.web.rest.feature", + "GRAVITINO_INITIAL_ADMIN_PASSWORD", + "automountServiceAccountToken", + "ClusterIP", + "emptyDir", + ): + if marker not in rendered: + errors.append(f"Gravitino identity manifest is missing marker: {marker}") + if "gravitino.authenticators = simple" in rendered: + errors.append("Gravitino identity manifest may not enable the simple authenticator") + return errors + + +def build_contract_report( + profile_path: Path = DEFAULT_PROFILE_PATH, + wrapper_path: Path = DEFAULT_WRAPPER_PATH, +) -> dict[str, Any]: + errors: list[str] = [] + profile: GravitinoIdentityProfile | None = None + try: + profile = load_profile(profile_path) + except MetadataFabricGravitinoIdentityError as exc: + errors.append(str(exc)) + errors.extend(_validate_manifest()) + + try: + wrapper = wrapper_path.resolve().read_text(encoding="utf-8") + for marker in ("set -euo pipefail", "metadata_fabric_gravitino_identity"): + if marker not in wrapper: + errors.append(f"Gravitino identity wrapper is missing marker: {marker}") + except OSError as exc: + errors.append(f"Gravitino identity wrapper is invalid: {type(exc).__name__}") + + files: dict[str, dict[str, str]] = {} + paths = [Path(__file__).resolve(), profile_path.resolve(), wrapper_path.resolve()] + paths.extend(sorted(MANIFEST_DIR.glob("*.yaml"))) + for path in paths: + if not path.is_file(): + continue + try: + relative = path.relative_to(REPO_ROOT).as_posix() + except ValueError: + relative = path.name + files[relative] = {"path": relative, "sha256": recovery._file_sha256(path)} + + stable = { + "schema": CONTRACT_SCHEMA, + "context": CONTEXT, + "source_namespace": SOURCE_NAMESPACE, + "rehearsal_namespace": REHEARSAL_NAMESPACE, + "gravitino_version": "1.3.0", + "authentication": { + "authenticator": profile.runtime.authenticator if profile else None, + "access_control_enabled": ( + profile.runtime.access_control_enabled if profile else None + ), + "idp_extension": profile.runtime.idp_extension if profile else None, + "simple_authenticator_trusted": False, + "built_in_idp_scope": "local_poc_only", + }, + "role_securable_objects": ( + _profile_securable_objects(profile) if profile else None + ), + "local_static_contract_verified": not errors, + "local_gravitino_basic_identity_verified": False, + "local_gravitino_minimum_privilege_verified": False, + "gravitino_authentication_verified": False, + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "production_identity_verified": False, + "production_ready": False, + "files": files, + "errors": errors, + } + return {**stable, "contract_fingerprint": recovery._canonical_sha256(stable)} + + +class _Kubectl: + def __init__(self, context: str) -> None: + self.context = context + + def run( + self, + args: list[str], + *, + input_text: str | None = None, + expected: frozenset[int] = frozenset({0}), + timeout: int = 300, + label: str, + ) -> subprocess.CompletedProcess[str]: + try: + completed = subprocess.run( + ["kubectl", "--context", self.context, *args], + input=input_text, + capture_output=True, + check=False, + text=True, + timeout=timeout, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise MetadataFabricGravitinoIdentityError( + f"kubectl failed during {label}" + ) from exc + if completed.returncode not in expected: + raise MetadataFabricGravitinoIdentityError( + f"kubectl returned {completed.returncode} during {label}" + ) + return completed + + def get_json( + self, + args: list[str], + *, + allow_not_found: bool = False, + label: str, + ) -> dict[str, Any] | None: + expected = frozenset({0, 1}) if allow_not_found else frozenset({0}) + completed = self.run( + [*args, "-o", "json"], expected=expected, timeout=60, label=label + ) + if completed.returncode != 0: + return None + try: + value = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise MetadataFabricGravitinoIdentityError( + f"kubectl returned invalid JSON during {label}" + ) from exc + if not isinstance(value, dict): + raise MetadataFabricGravitinoIdentityError( + f"kubectl returned a non-object during {label}" + ) + return value + + +class IsolatedGravitinoRuntime: + """Own and remove the temporary namespace used by the identity rehearsal.""" + + def __init__(self, profile: GravitinoIdentityProfile) -> None: + self.profile = profile + self.kubectl = _Kubectl(profile.cluster.context) + self.owned_namespace = False + self.schema_sha256: str | None = None + + def _runtime_inputs( + self, + admin_material: SecretStr, + database_material: SecretStr, + ) -> str: + source = self.kubectl.get_json( + [ + "-n", + self.profile.cluster.source_namespace, + "get", + "configmap", + self.profile.cluster.source_schema_configmap, + ], + label="source schema lookup", + ) + assert source is not None + schema_sql = _mapping(source.get("data")).get("001-schema.sql") + if not isinstance(schema_sql, str): + raise MetadataFabricGravitinoIdentityError( + "verified Gravitino PostgreSQL schema is unavailable" + ) + self.schema_sha256 = _sha256_text(schema_sql) + if self.schema_sha256 != GRAVITINO_SCHEMA_SHA256: + raise MetadataFabricGravitinoIdentityError( + "Gravitino PostgreSQL schema checksum drift" + ) + resources = { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "gravitino-identity-runtime", + "namespace": self.profile.cluster.rehearsal_namespace, + }, + "type": "Opaque", + "stringData": { + "admin-password": admin_material.get_secret_value(), + "database-password": database_material.get_secret_value(), + }, + }, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "gravitino-identity-schema", + "namespace": self.profile.cluster.rehearsal_namespace, + }, + "data": {"001-schema.sql": schema_sql}, + }, + ], + } + return json.dumps(resources, ensure_ascii=True, separators=(",", ":")) + + def start( + self, + *, + admin_material: SecretStr, + database_material: SecretStr, + ) -> dict[str, Any]: + existing = self.kubectl.get_json( + ["get", "namespace", self.profile.cluster.rehearsal_namespace], + allow_not_found=True, + label="rehearsal namespace preflight", + ) + if existing is not None: + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity rehearsal namespace already exists" + ) + self.kubectl.run( + ["apply", "-f", str(MANIFEST_DIR / "namespace.yaml")], + label="rehearsal namespace apply", + ) + self.owned_namespace = True + runtime_inputs = self._runtime_inputs(admin_material, database_material) + self.kubectl.run( + ["apply", "-f", "-"], + input_text=runtime_inputs, + label="ephemeral runtime input apply", + ) + self.kubectl.run( + ["apply", "-k", str(MANIFEST_DIR)], + label="isolated Gravitino runtime apply", + ) + namespace = self.profile.cluster.rehearsal_namespace + for workload in ( + "statefulset/gravitino-identity-postgresql", + "statefulset/gravitino-identity", + ): + self.kubectl.run( + [ + "-n", + namespace, + "rollout", + "status", + workload, + "--timeout=10m", + ], + timeout=660, + label=f"{workload} rollout", + ) + return self.observe() + + def observe(self) -> dict[str, Any]: + namespace_name = self.profile.cluster.rehearsal_namespace + namespace = self.kubectl.get_json( + ["get", "namespace", namespace_name], label="namespace observation" + ) + service = self.kubectl.get_json( + ["-n", namespace_name, "get", "service", "gravitino-identity"], + label="service observation", + ) + workload = self.kubectl.get_json( + [ + "-n", + namespace_name, + "get", + "statefulset", + "gravitino-identity", + ], + label="Gravitino workload observation", + ) + database = self.kubectl.get_json( + [ + "-n", + namespace_name, + "get", + "statefulset", + "gravitino-identity-postgresql", + ], + label="PostgreSQL workload observation", + ) + assert namespace is not None and service is not None + assert workload is not None and database is not None + workload_spec = _mapping(_mapping(workload.get("spec")).get("template")) + workload_pod = _mapping(workload_spec.get("spec")) + containers = workload_pod.get("containers") + container = _mapping(containers[0]) if isinstance(containers, list) else {} + database_spec = _mapping(_mapping(database.get("spec")).get("template")) + database_pod = _mapping(database_spec.get("spec")) + database_containers = database_pod.get("containers") + database_container = ( + _mapping(database_containers[0]) + if isinstance(database_containers, list) + else {} + ) + data_volume = next( + ( + _mapping(item) + for item in database_pod.get("volumes", []) + if _mapping(item).get("name") == "data" + ), + {}, + ) + return { + "context": self.profile.cluster.context, + "namespace": { + "name": _mapping(namespace.get("metadata")).get("name"), + "uid": _mapping(namespace.get("metadata")).get("uid"), + }, + "service": { + "name": _mapping(service.get("metadata")).get("name"), + "uid": _mapping(service.get("metadata")).get("uid"), + "type": _mapping(service.get("spec")).get("type"), + }, + "gravitino": { + "kind": workload.get("kind"), + "name": _mapping(workload.get("metadata")).get("name"), + "uid": _mapping(workload.get("metadata")).get("uid"), + "image": container.get("image"), + "service_account": workload_pod.get("serviceAccountName"), + "service_account_automount_disabled": ( + workload_pod.get("automountServiceAccountToken") is False + ), + "ready_replicas": _mapping(workload.get("status")).get( + "readyReplicas", 0 + ), + }, + "postgresql": { + "kind": database.get("kind"), + "name": _mapping(database.get("metadata")).get("name"), + "uid": _mapping(database.get("metadata")).get("uid"), + "image": database_container.get("image"), + "service_account": database_pod.get("serviceAccountName"), + "service_account_automount_disabled": ( + database_pod.get("automountServiceAccountToken") is False + ), + "ready_replicas": _mapping(database.get("status")).get( + "readyReplicas", 0 + ), + "ephemeral_data_volume": "emptyDir" in data_volume, + }, + "source_schema_sha256": self.schema_sha256, + } + + def cleanup(self) -> dict[str, Any]: + deleted = False + if self.owned_namespace: + try: + self.kubectl.run( + [ + "delete", + "namespace", + self.profile.cluster.rehearsal_namespace, + "--wait=true", + "--timeout=5m", + ], + timeout=330, + label="rehearsal namespace cleanup", + ) + deleted = True + finally: + self.owned_namespace = False + absent = ( + self.kubectl.get_json( + ["get", "namespace", self.profile.cluster.rehearsal_namespace], + allow_not_found=True, + label="rehearsal namespace cleanup verification", + ) + is None + ) + return { + "namespace_delete_completed": deleted, + "namespace_absent": absent, + "provider_objects_retained": False, + } + + +def _json_object(response: httpx.Response, label: str) -> dict[str, Any]: + try: + value = response.json() + except ValueError as exc: + raise ProviderRequestError(f"{label} returned invalid JSON") from exc + if not isinstance(value, dict): + raise ProviderRequestError(f"{label} response is not an object") + return value + + +class _BasicApi: + def __init__( + self, + *, + base_url: str, + username: str, + material: SecretStr, + transport: httpx.BaseTransport | None = None, + ) -> None: + self._client = httpx.Client( + base_url=base_url.rstrip("/") + "/", + auth=httpx.BasicAuth(username, material.get_secret_value()), + headers={ + "Accept": "application/vnd.gravitino.v1+json", + "Content-Type": "application/json", + }, + timeout=30.0, + transport=transport, + ) + + def close(self) -> None: + self._client.close() + + def request( + self, + method: str, + path: str, + *, + json_body: Mapping[str, Any] | None = None, + params: Mapping[str, str] | None = None, + expected: frozenset[int] = frozenset({200}), + label: str, + ) -> tuple[int, dict[str, Any] | None]: + try: + response = self._client.request( + method, + path, + json=dict(json_body) if json_body is not None else None, + params=params, + ) + except httpx.HTTPError as exc: + raise ProviderRequestError(f"{label} request failed") from exc + if response.status_code not in expected: + raise ProviderRequestError( + f"{label} returned unexpected status {response.status_code}" + ) + if response.status_code in {401, 403, 404} or not response.content: + return response.status_code, None + return response.status_code, _json_object(response, label) + + +def _response_entity( + payload: Mapping[str, Any] | None, key: str, label: str +) -> dict[str, Any]: + entity = _mapping(_mapping(payload).get(key)) + if not entity: + raise ProviderRequestError(f"{label} omitted {key}") + return dict(entity) + + +def _normalize_securable_objects(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + normalized: list[dict[str, Any]] = [] + for item in value: + source = _mapping(item) + privileges = source.get("privileges") + normalized_privileges = [] + if isinstance(privileges, list): + for privilege in privileges: + entry = _mapping(privilege) + normalized_privileges.append( + { + "name": str(entry.get("name", "")).upper(), + "condition": str(entry.get("condition", "")).upper(), + } + ) + normalized.append( + { + "fullName": source.get("fullName"), + "type": str(source.get("type", "")).upper(), + "privileges": sorted( + normalized_privileges, key=lambda entry: str(entry["name"]) + ), + } + ) + return sorted(normalized, key=lambda entry: str(entry["fullName"])) + + +class GravitinoIdentityRehearsal: + """Exercise one Basic IdP user with a schema-bounded table-create role.""" + + def __init__( + self, + *, + base_url: str, + admin_name: str, + admin_material: SecretStr, + transport: httpx.BaseTransport | None = None, + ) -> None: + self.base_url = base_url + self.transport = transport + self.admin = _BasicApi( + base_url=base_url, + username=admin_name, + material=admin_material, + transport=transport, + ) + self.clients: list[_BasicApi] = [] + + def close(self) -> None: + for client in self.clients: + client.close() + self.clients.clear() + self.admin.close() + + def _user_api(self, name: str, material: SecretStr) -> _BasicApi: + client = _BasicApi( + base_url=self.base_url, + username=name, + material=material, + transport=self.transport, + ) + self.clients.append(client) + return client + + @staticmethod + def _catalog_path(profile: GravitinoIdentityProfile, catalog: str) -> str: + return f"metalakes/{quote(profile.scope.metalake)}/catalogs/{quote(catalog)}" + + @staticmethod + def _schema_path(profile: GravitinoIdentityProfile) -> str: + return ( + f"metalakes/{quote(profile.scope.metalake)}/catalogs/" + f"{quote(profile.scope.catalog)}/schemas/{quote(profile.scope.schema_name)}" + ) + + @classmethod + def _table_path(cls, profile: GravitinoIdentityProfile) -> str: + return f"{cls._schema_path(profile)}/tables/{quote(profile.scope.table)}" + + def execute( + self, + profile: GravitinoIdentityProfile, + *, + initial_material: SecretStr, + rotated_material: SecretStr, + ) -> dict[str, Any]: + admin_status, version_payload = self.admin.request( + "GET", "version", label="Gravitino service-admin authentication" + ) + version = _mapping(_mapping(version_payload).get("version")).get("version") + if version is None: + version = _mapping(version_payload).get("version") + + unregistered = self._user_api(profile.identity.user, initial_material) + unregistered_status, _payload = unregistered.request( + "GET", + "version", + expected=frozenset({200, 401}), + label="Gravitino unregistered principal probe", + ) + + _status, metalake_payload = self.admin.request( + "POST", + "metalakes", + json_body={ + "name": profile.scope.metalake, + "comment": "Ephemeral GDA authenticated identity rehearsal", + "properties": {"gda.environment": "local_identity_rehearsal"}, + }, + label="Gravitino metalake create", + ) + metalake = _response_entity(metalake_payload, "metalake", "metalake create") + _status, catalog_payload = self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/catalogs", + json_body={ + "name": profile.scope.catalog, + "type": "RELATIONAL", + "provider": "lakehouse-iceberg", + "comment": "Ephemeral authenticated table projection catalog", + "properties": { + "catalog-backend": "memory", + "uri": "file:///tmp/gda-identity", + "warehouse": "file:///tmp/gda-identity", + }, + }, + label="Gravitino catalog create", + ) + catalog = _response_entity(catalog_payload, "catalog", "catalog create") + _status, schema_payload = self.admin.request( + "POST", + f"{self._catalog_path(profile, profile.scope.catalog)}/schemas", + json_body={ + "name": profile.scope.schema_name, + "comment": "Ephemeral bounded table projection schema", + "properties": {}, + }, + label="Gravitino schema create", + ) + schema = _response_entity(schema_payload, "schema", "schema create") + + _status, idp_user_payload = self.admin.request( + "POST", + "idp/users", + json_body={ + "user": profile.identity.user, + "password": initial_material.get_secret_value(), + }, + label="Gravitino built-in IdP user create", + ) + _status, user_payload = self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/users", + json_body={"name": profile.identity.user}, + label="Gravitino metalake user register", + ) + user = _response_entity(user_payload, "user", "metalake user register") + + _status, role_payload = self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/roles", + json_body={ + "name": profile.identity.role, + "properties": {"gda.scope": "bounded_table_projection"}, + "securableObjects": _expected_securable_objects(), + }, + label="Gravitino bounded role create", + ) + role = _response_entity(role_payload, "role", "role create") + _status, granted_payload = self.admin.request( + "PUT", + ( + f"metalakes/{quote(profile.scope.metalake)}/permissions/users/" + f"{quote(profile.identity.user)}/grant" + ), + json_body={"roleNames": [profile.identity.role]}, + label="Gravitino role grant", + ) + granted_user = _response_entity(granted_payload, "user", "role grant") + _status, role_read_payload = self.admin.request( + "GET", + ( + f"metalakes/{quote(profile.scope.metalake)}/roles/" + f"{quote(profile.identity.role)}" + ), + label="Gravitino role readback", + ) + role_read = _response_entity(role_read_payload, "role", "role readback") + + initial_api = self._user_api(profile.identity.user, initial_material) + initial_status, _initial_payload = initial_api.request( + "GET", "version", label="Gravitino bounded user authentication" + ) + create_status, table_payload = initial_api.request( + "POST", + f"{self._schema_path(profile)}/tables", + json_body={ + "name": profile.scope.table, + "comment": "Ephemeral authenticated table-create probe", + "columns": [ + { + "name": "probe_id", + "type": "string", + "nullable": False, + "comment": "Identity probe identifier", + } + ], + "properties": {"gda.identity_probe": "true"}, + }, + label="Gravitino allowed table create probe", + ) + table = _response_entity(table_payload, "table", "table create") + read_status, table_read_payload = initial_api.request( + "GET", self._table_path(profile), label="Gravitino table readback" + ) + table_read = _response_entity(table_read_payload, "table", "table readback") + + denied_status, _denied_payload = initial_api.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/catalogs", + json_body={ + "name": profile.scope.denied_catalog, + "type": "RELATIONAL", + "provider": "lakehouse-iceberg", + "comment": "Must not be created by the bounded user", + "properties": { + "catalog-backend": "memory", + "uri": "file:///tmp/gda-identity-denied", + "warehouse": "file:///tmp/gda-identity-denied", + }, + }, + expected=frozenset({200, 403}), + label="Gravitino denied catalog create probe", + ) + + self.admin.request( + "PUT", + f"idp/users/{quote(profile.identity.user)}", + json_body={"password": rotated_material.get_secret_value()}, + label="Gravitino built-in IdP login rotation", + ) + old_after_rotation, _payload = initial_api.request( + "GET", + "version", + expected=frozenset({200, 401}), + label="Gravitino old login after rotation", + ) + rotated_api = self._user_api(profile.identity.user, rotated_material) + rotated_status, rotated_table_payload = rotated_api.request( + "GET", + self._table_path(profile), + expected=frozenset({200, 401, 403}), + label="Gravitino rotated login", + ) + rotated_table = ( + _response_entity(rotated_table_payload, "table", "rotated table readback") + if rotated_status == 200 + else {} + ) + self.admin.request( + "DELETE", + f"idp/users/{quote(profile.identity.user)}", + label="Gravitino built-in IdP user revocation", + ) + after_revocation, _payload = rotated_api.request( + "GET", + "version", + expected=frozenset({200, 401}), + label="Gravitino revoked login", + ) + idp_lookup_status, _payload = self.admin.request( + "GET", + f"idp/users/{quote(profile.identity.user)}", + expected=frozenset({200, 404}), + label="Gravitino built-in IdP user absence", + ) + + return { + "version": version, + "configuration": { + "authenticator": profile.runtime.authenticator, + "access_control_enabled": profile.runtime.access_control_enabled, + "idp_extension": profile.runtime.idp_extension, + "built_in_idp_scope": "local_poc_only", + "simple_authenticator_trusted": False, + }, + "authentication": { + "service_admin_status": admin_status, + "unregistered_principal_status": unregistered_status, + "bounded_user_status": initial_status, + "material_recorded": False, + }, + "principal": { + "name": user.get("name"), + "granted_name": granted_user.get("name"), + "roles": sorted(granted_user.get("roles", [])), + "is_service_admin": False, + }, + "role": { + "name": role.get("name"), + "readback_name": role_read.get("name"), + "securable_objects": _normalize_securable_objects( + role_read.get("securableObjects") + ), + }, + "bootstrap": { + "metalake": metalake.get("name"), + "catalog": catalog.get("name"), + "schema": schema.get("name"), + "service_admin_used": True, + "minimum_privilege": False, + }, + "allowed_probe": { + "operation": "CREATE_TABLE", + "resource": "SCHEMA", + "create_status": create_status, + "read_status": read_status, + "table": table.get("name"), + "readback_table": table_read.get("name"), + "rotated_readback_table": rotated_table.get("name"), + "full_name": ( + f"{profile.scope.catalog}.{profile.scope.schema_name}." + f"{profile.scope.table}" + ), + }, + "denied_probe": { + "operation": "CREATE_CATALOG", + "resource": "METALAKE", + "catalog": profile.scope.denied_catalog, + "status": denied_status, + }, + "login_lifecycle": { + "rotation_mode": profile.identity.login_rotation, + "old_after_rotation_status": old_after_rotation, + "rotated_authentication_status": rotated_status, + "revocation_mode": profile.identity.revocation, + "after_revocation_status": after_revocation, + "idp_principal_absent": idp_lookup_status == 404, + "material_recorded": False, + }, + "idp_create_response_present": bool(idp_user_payload), + } + + +def build_evidence(observation: Mapping[str, Any]) -> dict[str, Any]: + errors: list[str] = [] + try: + ingestion_replay._reject_sensitive_fields(observation) + except ValueError: + errors.append("Gravitino identity observation contains sensitive material") + if observation.get("schema") != OBSERVATION_SCHEMA: + errors.append("Gravitino identity observation schema does not match") + + contract = _mapping(observation.get("contract")) + if ( + contract.get("local_static_contract_verified") is not True + or not _valid_sha256(contract.get("contract_fingerprint")) + ): + errors.append("Gravitino identity static contract is not bound") + + runtime = _mapping(observation.get("runtime")) + gravitino_runtime = _mapping(runtime.get("gravitino")) + postgresql_runtime = _mapping(runtime.get("postgresql")) + if ( + runtime.get("context") != CONTEXT + or _mapping(runtime.get("namespace")).get("name") != REHEARSAL_NAMESPACE + or not _valid_uuid(_mapping(runtime.get("namespace")).get("uid")) + or _mapping(runtime.get("service")).get("name") != "gravitino-identity" + or _mapping(runtime.get("service")).get("type") != "ClusterIP" + or not _valid_uuid(_mapping(runtime.get("service")).get("uid")) + or runtime.get("source_schema_sha256") != GRAVITINO_SCHEMA_SHA256 + or gravitino_runtime.get("kind") != "StatefulSet" + or gravitino_runtime.get("name") != "gravitino-identity" + or gravitino_runtime.get("image") != "gda/gravitino:1.3.0-local-arm64" + or gravitino_runtime.get("service_account") != "gravitino-identity" + or gravitino_runtime.get("service_account_automount_disabled") is not True + or gravitino_runtime.get("ready_replicas") != 1 + or postgresql_runtime.get("kind") != "StatefulSet" + or postgresql_runtime.get("name") != "gravitino-identity-postgresql" + or postgresql_runtime.get("image") != "postgres:16.10-bookworm" + or postgresql_runtime.get("service_account") + != "gravitino-identity-postgresql" + or postgresql_runtime.get("service_account_automount_disabled") is not True + or postgresql_runtime.get("ready_replicas") != 1 + or postgresql_runtime.get("ephemeral_data_volume") is not True + ): + errors.append("isolated Gravitino identity runtime does not match") + + provider = _mapping(observation.get("gravitino")) + configuration = _mapping(provider.get("configuration")) + if configuration != { + "authenticator": "basic", + "access_control_enabled": True, + "idp_extension": "org.apache.gravitino.idp.web.rest.feature", + "built_in_idp_scope": "local_poc_only", + "simple_authenticator_trusted": False, + }: + errors.append("Gravitino Basic IdP configuration does not match") + if provider.get("version") != "1.3.0": + errors.append("Gravitino identity provider version does not match") + + authentication = _mapping(provider.get("authentication")) + if authentication != { + "service_admin_status": 200, + "unregistered_principal_status": 401, + "bounded_user_status": 200, + "material_recorded": False, + }: + errors.append("Gravitino Basic authentication was not enforced") + + principal = _mapping(provider.get("principal")) + if ( + principal.get("name") != "gda-metadata-projection" + or principal.get("granted_name") != "gda-metadata-projection" + or principal.get("roles") != ["gda-table-projection"] + or principal.get("is_service_admin") is not False + ): + errors.append("Gravitino bounded principal does not match") + + role = _mapping(provider.get("role")) + if ( + role.get("name") != "gda-table-projection" + or role.get("readback_name") != "gda-table-projection" + or role.get("securable_objects") != _expected_securable_objects() + ): + errors.append("Gravitino role exceeds the bounded table-create scope") + + bootstrap = _mapping(provider.get("bootstrap")) + if bootstrap != { + "metalake": "gda_identity", + "catalog": "lakehouse", + "schema": "published", + "service_admin_used": True, + "minimum_privilege": False, + }: + errors.append("Gravitino service-admin bootstrap boundary is missing") + + allowed = _mapping(provider.get("allowed_probe")) + if ( + allowed.get("operation") != "CREATE_TABLE" + or allowed.get("resource") != "SCHEMA" + or allowed.get("create_status") != 200 + or allowed.get("read_status") != 200 + or allowed.get("table") != "gda_identity_probe" + or allowed.get("readback_table") != "gda_identity_probe" + or allowed.get("rotated_readback_table") != "gda_identity_probe" + or allowed.get("full_name") != "lakehouse.published.gda_identity_probe" + ): + errors.append("Gravitino allowed table-create probe did not pass") + + denied = _mapping(provider.get("denied_probe")) + if denied != { + "operation": "CREATE_CATALOG", + "resource": "METALAKE", + "catalog": "unauthorized_catalog", + "status": 403, + }: + errors.append("Gravitino catalog-create denial was not enforced") + + lifecycle = _mapping(provider.get("login_lifecycle")) + if lifecycle != { + "rotation_mode": "administrator_reset", + "old_after_rotation_status": 401, + "rotated_authentication_status": 200, + "revocation_mode": "idp_user_delete", + "after_revocation_status": 401, + "idp_principal_absent": True, + "material_recorded": False, + }: + errors.append("Gravitino login rotation or revocation did not pass") + if provider.get("idp_create_response_present") is not True: + errors.append("Gravitino built-in IdP user creation was not observed") + + runtime_checks = _mapping(observation.get("runtime_checks")) + if runtime_checks != { + "namespace_delete_completed": True, + "namespace_absent": True, + "provider_objects_retained": False, + "all_port_forwards_stopped": True, + "material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }: + errors.append("Gravitino identity runtime cleanup is incomplete") + + verified = not errors + stable = { + "schema": EVIDENCE_SCHEMA, + "status": "local_gravitino_basic_identity_verified" if verified else "blocked", + "observation": dict(observation), + "errors": errors, + "local_gravitino_basic_identity_verified": verified, + "local_gravitino_minimum_privilege_verified": verified, + "local_gravitino_login_rotation_verified": verified, + "local_gravitino_revocation_verified": verified, + "gravitino_authentication_verified": False, + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "production_identity_verified": False, + "production_ready": False, + } + return {**stable, "evidence_fingerprint": recovery._canonical_sha256(stable)} + + +def verify_evidence_integrity(evidence: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if evidence.get("schema") != EVIDENCE_SCHEMA: + errors.append("Gravitino identity evidence schema does not match") + expected = build_evidence(_mapping(evidence.get("observation"))) + if dict(evidence) != expected: + errors.append("Gravitino identity evidence content or fingerprint drifted") + for claim in ( + "gravitino_authentication_verified", + "provider_minimum_privilege_verified", + "protected_workload_identity_verified", + "oidc_verified", + "tls_verified", + "production_identity_verified", + "production_ready", + ): + if evidence.get(claim) is not False: + errors.append(f"Gravitino identity evidence may not claim {claim}") + if evidence.get("local_gravitino_basic_identity_verified") is not True: + errors.append("local Gravitino Basic identity is not verified") + return errors + + +def run_live_rehearsal( + profile_path: Path = DEFAULT_PROFILE_PATH, +) -> dict[str, Any]: + profile = load_profile(profile_path) + contract = build_contract_report(profile_path) + if contract.get("local_static_contract_verified") is not True: + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity static contract is invalid" + ) + + admin_material = SecretStr(secrets.token_urlsafe(24)) + database_material = SecretStr(secrets.token_urlsafe(24)) + initial_material = SecretStr(secrets.token_urlsafe(24)) + rotated_material = SecretStr(secrets.token_urlsafe(24)) + runtime = IsolatedGravitinoRuntime(profile) + forward: provider_metrics._PortForward | None = None + rehearsal: GravitinoIdentityRehearsal | None = None + runtime_observation: dict[str, Any] | None = None + result: dict[str, Any] | None = None + forward_stopped = False + cleanup: dict[str, Any] = { + "namespace_delete_completed": False, + "namespace_absent": False, + "provider_objects_retained": True, + } + try: + runtime_observation = runtime.start( + admin_material=admin_material, + database_material=database_material, + ) + forward = provider_metrics._PortForward( + kubectl="kubectl", + context=profile.cluster.context, + namespace=profile.cluster.rehearsal_namespace, + service=profile.runtime.service, + target_port=profile.runtime.service_port, + ) + forward.start() + rehearsal = GravitinoIdentityRehearsal( + base_url=f"http://127.0.0.1:{forward.local_port}/api", + admin_name=profile.identity.service_admin, + admin_material=admin_material, + ) + result = rehearsal.execute( + profile, + initial_material=initial_material, + rotated_material=rotated_material, + ) + finally: + if rehearsal is not None: + rehearsal.close() + if forward is not None: + forward_stopped = forward.stop() + cleanup = runtime.cleanup() + + if runtime_observation is None or result is None: + raise MetadataFabricGravitinoIdentityError( + "Gravitino identity rehearsal did not produce an outcome" + ) + observation = { + "schema": OBSERVATION_SCHEMA, + "observed_at": datetime.now(UTC).isoformat(), + "contract": { + "contract_fingerprint": contract["contract_fingerprint"], + "local_static_contract_verified": True, + }, + "runtime": runtime_observation, + "gravitino": result, + "runtime_checks": { + **cleanup, + "all_port_forwards_stopped": forward_stopped, + "material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }, + } + return build_evidence(observation) + + +def build_validation_report( + *, + profile_path: Path = DEFAULT_PROFILE_PATH, + evidence_path: Path = DEFAULT_EVIDENCE_PATH, +) -> dict[str, Any]: + contract = build_contract_report(profile_path) + errors = list(contract["errors"]) + evidence: dict[str, Any] | None = None + try: + value = json.loads(evidence_path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("Gravitino identity evidence must be an object") + evidence = value + errors.extend(verify_evidence_integrity(evidence)) + observed_contract = _mapping( + _mapping(evidence.get("observation")).get("contract") + ).get("contract_fingerprint") + if observed_contract != contract.get("contract_fingerprint"): + errors.append("Gravitino identity evidence contract fingerprint drift") + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + errors.append(f"Gravitino identity evidence is invalid: {type(exc).__name__}") + verified = not errors + return { + "schema": VALIDATION_SCHEMA, + "local_static_contract_verified": contract["local_static_contract_verified"], + "local_gravitino_basic_identity_verified": ( + verified + and evidence is not None + and evidence.get("local_gravitino_basic_identity_verified") is True + ), + "local_gravitino_minimum_privilege_verified": ( + verified + and evidence is not None + and evidence.get("local_gravitino_minimum_privilege_verified") is True + ), + "gravitino_authentication_verified": False, + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "production_identity_verified": False, + "production_ready": False, + "contract_fingerprint": contract["contract_fingerprint"], + "evidence_fingerprint": evidence.get("evidence_fingerprint") if evidence else None, + "errors": errors, + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + validate.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + rehearse = subparsers.add_parser("rehearse") + rehearse.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + rehearse.add_argument("--evidence-out", type=Path, required=True) + verify = subparsers.add_parser("verify") + verify.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + args = parser.parse_args(argv) + try: + if args.command == "validate": + report = build_validation_report( + profile_path=args.profile, evidence_path=args.evidence + ) + print(json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not report["errors"] else 1 + if args.command == "verify": + value = json.loads(args.evidence.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("Gravitino identity evidence must be an object") + errors = verify_evidence_integrity(value) + print(json.dumps({"verified": not errors, "errors": errors}, indent=2)) + return 0 if not errors else 1 + evidence = run_live_rehearsal(args.profile) + _write_json(args.evidence_out, evidence) + print(json.dumps(evidence, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not evidence["errors"] else 1 + except ( + KeyError, + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + MetadataFabricGravitinoIdentityError, + KeyboardInterrupt, + ) as exc: + print(f"metadata fabric Gravitino identity: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/metadata_fabric_identity_gate.py b/data_agent/metadata_fabric_identity_gate.py new file mode 100644 index 00000000..3e60673c --- /dev/null +++ b/data_agent/metadata_fabric_identity_gate.py @@ -0,0 +1,882 @@ +"""Evaluate the fail-closed production Metadata Fabric identity gate. + +The checked-in profile freezes the decisions and protected-environment checks +required to promote the local OpenMetadata and Gravitino identity rehearsals. +Missing decisions are explicit blockers. A production identity result requires +a fresh attestation bound to the exact profile and all derived bindings. This +module deploys nothing, accepts no credentials, and never claims that the whole +platform is production ready. +""" + +from __future__ import annotations + +import argparse +import ipaddress +import json +import re +import sys +from collections.abc import Mapping +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +import yaml + +from . import metadata_fabric_recovery_rehearsal as recovery + + +PROFILE_SCHEMA = "gda.metadata_fabric_identity_production_profile.v1" +ATTESTATION_SCHEMA = "gda.metadata_fabric_identity_attestation.v1" +REPORT_SCHEMA = "gda.metadata_fabric_identity_readiness_report.v1" +ENVIRONMENT = "production" + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILE_PATH = REPO_ROOT / "config/metadata-fabric-identity.production.yaml" + +EXPECTED_PROVIDERS = {"openmetadata": "1.13.1", "gravitino": "1.3.0"} +EXPECTED_LOCAL_EVIDENCE = { + "openmetadata": { + "path": "docs/evidence/metadata-fabric-provider-identity-2026-07-28.json", + "evidence_fingerprint": ( + "61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f" + ), + }, + "gravitino": { + "path": "docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json", + "evidence_fingerprint": ( + "f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd" + ), + }, +} +EXPECTED_AUTHORIZATION = { + "openmetadata": { + "mandatory_roles": ["DefaultBotRole"], + "project_role": "GdaMetadataTableProjectionRole", + "project_permissions": [{"resource": "table", "operations": ["Create"]}], + "denied_probe": {"resource": "policy", "operation": "Create"}, + }, + "gravitino": { + "role": "gda-table-projection", + "securable_objects": [ + { + "full_name": "lakehouse", + "type": "CATALOG", + "privileges": ["USE_CATALOG"], + }, + { + "full_name": "lakehouse.published", + "type": "SCHEMA", + "privileges": ["CREATE_TABLE", "USE_SCHEMA"], + }, + ], + "denied_probe": {"resource": "METALAKE", "operation": "CREATE_CATALOG"}, + }, +} +ALLOWED_INTEGRATION_MODES = { + "openmetadata": {"provider_native_oidc", "identity_aware_proxy"}, + "gravitino": {"custom_oidc_authenticator", "identity_aware_proxy"}, +} +ALLOWED_CATALOG_BACKENDS = {"iceberg_rest", "jdbc"} +EXPECTED_ATTESTATION_CHECKS = { + "oidc_discovery", + "token_exchange", + "workload_subject_binding", + "tenant_claim_binding", + "openmetadata_authenticated_allow", + "openmetadata_administrative_deny", + "gravitino_authenticated_allow", + "gravitino_administrative_deny", + "provider_direct_access_denied", + "static_credential_absence", + "credential_rotation", + "credential_revocation", + "tls_transport", + "mtls_internal_hops", + "persistent_catalog_restart", + "cross_tenant_denial", + "audit_log_delivery", + "rollback_rehearsal", +} +EXPECTED_CLAIMS = { + "identity_decision_frozen", + "provider_minimum_privilege_verified", + "protected_workload_identity_verified", + "oidc_verified", + "tls_verified", + "credential_rotation_verified", + "credential_revocation_verified", + "persistent_catalog_identity_binding_verified", + "production_identity_verified", + "production_identity_gate_passed", + "production_ready", +} +REPORT_CLAIMS = { + "provider_minimum_privilege_verified", + "protected_workload_identity_verified", + "oidc_verified", + "tls_verified", + "credential_rotation_verified", + "credential_revocation_verified", + "persistent_catalog_identity_binding_verified", + "production_identity_verified", + "production_identity_gate_passed", +} +REPORT_INVENTORY = { + "schema", + "environment", + "profile_fingerprint", + "attestation_fingerprint", + "federation_fingerprint", + "provider_bindings_fingerprint", + "authorization_fingerprint", + "tls_fingerprint", + "catalog_fingerprint", + "tenancy_fingerprint", + "profile_valid", + "profile_errors", + "profile_blockers", + "ready_for_protected_verification", + "attestation_valid", + "attestation_errors", + *REPORT_CLAIMS, + "production_ready", + "report_fingerprint", +} + +SHA40_PATTERN = re.compile(r"^[0-9a-f]{40}$") +DNS_LABEL_PATTERN = re.compile(r"^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$") +REFERENCE_PATTERN = re.compile(r"^[a-z][a-z0-9+.-]*://[^\s]+$", re.IGNORECASE) +PINNED_OCI_PATTERN = re.compile(r"^oci://[^\s@]+@sha256:[0-9a-f]{64}$") +PLACEHOLDER_PATTERN = re.compile( + r"(^|[-_.:/])(pending|placeholder|replace|tbd|todo|changeme)([-_.:/]|$)|" + r"[<>]|\.example(?=[:/]|$)", + re.IGNORECASE, +) +SENSITIVE_KEY_PATTERN = re.compile( + r"(^|[-_.])(password|passwd|secret|client[-_.]?secret|private[-_.]?key|" + r"access[-_.]?key|access[-_.]?token|refresh[-_.]?token|id[-_.]?token|" + r"authorization[-_.]?header)($|[-_.])", + re.IGNORECASE, +) + + +class MetadataFabricIdentityGateError(RuntimeError): + """The production identity readiness contract failed closed.""" + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _load_yaml_object(path: Path) -> dict[str, Any]: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("YAML document is not an object") + return value + + +def _load_json_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("JSON document is not an object") + return value + + +def _inventory_errors( + value: Mapping[str, Any], expected: set[str], label: str +) -> list[str]: + return [] if set(value) == expected else [f"{label} inventory does not match"] + + +def _placeholder(value: Any) -> bool: + return not isinstance(value, str) or not value.strip() or bool( + PLACEHOLDER_PATTERN.search(value.strip()) + ) + + +def _reference(value: Any) -> bool: + return not _placeholder(value) and bool(REFERENCE_PATTERN.fullmatch(str(value))) + + +def _production_https_url(value: Any) -> bool: + if _placeholder(value): + return False + parsed = urlparse(str(value)) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + ): + return False + hostname = parsed.hostname.lower() + if hostname in {"localhost", "0.0.0.0", "::1"} or hostname.endswith( + (".localhost", ".local", ".svc", ".cluster.local") + ): + return False + if hostname in {"example.com", "example.net", "example.org"} or hostname.endswith( + (".example.com", ".example.net", ".example.org") + ): + return False + try: + if ipaddress.ip_address(hostname).is_private: + return False + except ValueError: + pass + return True + + +def _sensitive_paths(value: Any, path: tuple[str, ...] = ()) -> list[str]: + found: list[str] = [] + if isinstance(value, Mapping): + for key, nested in value.items(): + key_text = str(key) + child = (*path, key_text) + if SENSITIVE_KEY_PATTERN.search(key_text): + found.append(".".join(child)) + found.extend(_sensitive_paths(nested, child)) + elif isinstance(value, list): + for index, nested in enumerate(value): + found.extend(_sensitive_paths(nested, (*path, str(index)))) + return found + + +def _local_evidence_errors(value: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + errors.extend( + _inventory_errors(value, set(EXPECTED_LOCAL_EVIDENCE), "local evidence") + ) + for provider, expected in EXPECTED_LOCAL_EVIDENCE.items(): + item = _mapping(value.get(provider)) + errors.extend( + _inventory_errors( + item, {"path", "evidence_fingerprint"}, f"{provider} local evidence" + ) + ) + if dict(item) != expected: + errors.append(f"{provider} local identity evidence binding does not match") + continue + path = (REPO_ROOT / expected["path"]).resolve() + try: + path.relative_to(REPO_ROOT) + evidence = _load_json_object(path) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + errors.append(f"{provider} local identity evidence is unavailable") + continue + if evidence.get("evidence_fingerprint") != expected["evidence_fingerprint"]: + errors.append(f"{provider} local identity evidence fingerprint drifted") + if evidence.get("production_identity_verified") is not False: + errors.append(f"{provider} local evidence overclaims production identity") + if provider == "openmetadata": + if ( + evidence.get("local_openmetadata_bounded_identity_verified") is not True + or evidence.get("local_openmetadata_minimum_privilege_verified") is not True + ): + errors.append("OpenMetadata local identity evidence is not verified") + elif ( + evidence.get("local_gravitino_basic_identity_verified") is not True + or evidence.get("local_gravitino_minimum_privilege_verified") is not True + ): + errors.append("Gravitino local identity evidence is not verified") + return errors + + +def _profile_errors(profile: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if _sensitive_paths(profile): + errors.append("identity profile contains credential-bearing fields") + errors.extend( + _inventory_errors( + profile, + { + "schema", + "environment", + "scope", + "federation", + "providers", + "authorization", + "tls", + "catalog", + "tenancy", + "operations", + "claims", + }, + "identity profile", + ) + ) + if profile.get("schema") != PROFILE_SCHEMA or profile.get("environment") != ENVIRONMENT: + errors.append("identity profile schema or environment does not match") + + scope = _mapping(profile.get("scope")) + errors.extend( + _inventory_errors(scope, {"providers", "local_evidence"}, "identity scope") + ) + providers = _mapping(scope.get("providers")) + if set(providers) != set(EXPECTED_PROVIDERS): + errors.append("identity provider version inventory does not match") + for provider, version in EXPECTED_PROVIDERS.items(): + if _mapping(providers.get(provider)) != {"version": version}: + errors.append(f"{provider} version binding does not match") + errors.extend(_local_evidence_errors(_mapping(scope.get("local_evidence")))) + + federation = _mapping(profile.get("federation")) + federation_keys = { + "decision_status", + "issuer", + "discovery_uri", + "jwks_uri", + "audience", + "token_exchange_mode", + "trust_policy_reference", + "workload_subject_claim", + "tenant_claim", + "maximum_token_ttl_seconds", + } + errors.extend(_inventory_errors(federation, federation_keys, "identity federation")) + if federation.get("decision_status") not in {"pending", "approved"}: + errors.append("identity federation decision status is invalid") + for key in ("issuer", "discovery_uri", "jwks_uri"): + value = federation.get(key) + if value is not None and not _production_https_url(value): + errors.append(f"identity federation {key} is not a production HTTPS URL") + for key in ("audience", "trust_policy_reference"): + value = federation.get(key) + if value is not None and not _reference(value): + errors.append(f"identity federation {key} is invalid") + if federation.get("token_exchange_mode") not in { + None, + "oidc_workload_federation", + }: + errors.append("identity token exchange mode is invalid") + if federation.get("workload_subject_claim") != "sub": + errors.append("identity workload subject claim must remain sub") + if federation.get("tenant_claim") != "gda_tenant_id": + errors.append("identity tenant claim must remain gda_tenant_id") + maximum_ttl = federation.get("maximum_token_ttl_seconds") + if ( + not isinstance(maximum_ttl, int) + or isinstance(maximum_ttl, bool) + or not 300 <= maximum_ttl <= 900 + ): + errors.append("identity maximum token TTL must be between 300 and 900 seconds") + + provider_bindings = _mapping(profile.get("providers")) + errors.extend( + _inventory_errors(provider_bindings, set(EXPECTED_PROVIDERS), "provider binding") + ) + provider_keys = { + "integration_mode", + "authentication_component_reference", + "environment_binding", + "workload_identity_reference", + "kubernetes_service_account", + "namespace_template", + "direct_access_policy", + "credential_delivery", + "static_credentials_forbidden", + "rotation_mode", + "revocation_mode", + } + for provider in EXPECTED_PROVIDERS: + item = _mapping(provider_bindings.get(provider)) + errors.extend(_inventory_errors(item, provider_keys, f"{provider} identity binding")) + mode = item.get("integration_mode") + if mode is not None and mode not in ALLOWED_INTEGRATION_MODES[provider]: + errors.append(f"{provider} identity integration mode is invalid") + component = item.get("authentication_component_reference") + if component is not None and not PINNED_OCI_PATTERN.fullmatch(str(component)): + errors.append(f"{provider} authentication component must be digest-pinned OCI") + for key in ("environment_binding", "workload_identity_reference"): + value = item.get(key) + if value is not None and not _reference(value): + errors.append(f"{provider} {key} is invalid") + service_account = item.get("kubernetes_service_account") + if service_account is not None and not DNS_LABEL_PATTERN.fullmatch( + str(service_account) + ): + errors.append(f"{provider} Kubernetes ServiceAccount is invalid") + namespace = item.get("namespace_template") + if namespace is not None and ( + _placeholder(namespace) or "{tenant_id}" not in str(namespace) + ): + errors.append(f"{provider} namespace template must bind tenant_id") + expected_static = { + "direct_access_policy": "deny_except_attested_identity_path", + "credential_delivery": "short_lived_token_exchange", + "static_credentials_forbidden": True, + "rotation_mode": "automatic_before_expiry", + "revocation_mode": "idp_subject_disable", + } + for key, expected in expected_static.items(): + if item.get(key) != expected: + errors.append(f"{provider} identity control does not match: {key}") + + if _mapping(profile.get("authorization")) != EXPECTED_AUTHORIZATION: + errors.append("provider minimum-privilege authorization contract does not match") + + tls = _mapping(profile.get("tls")) + tls_keys = { + "required", + "minimum_version", + "openmetadata_endpoint", + "gravitino_endpoint", + "trust_bundle_reference", + "certificate_policy_reference", + "mtls_for_internal_hops", + } + errors.extend(_inventory_errors(tls, tls_keys, "identity TLS")) + if ( + tls.get("required") is not True + or tls.get("minimum_version") not in {"TLSv1.2", "TLSv1.3"} + or tls.get("mtls_for_internal_hops") is not True + ): + errors.append("identity TLS baseline does not match") + for key in ("openmetadata_endpoint", "gravitino_endpoint"): + value = tls.get(key) + if value is not None and not _production_https_url(value): + errors.append(f"identity TLS {key} is invalid") + for key in ("trust_bundle_reference", "certificate_policy_reference"): + value = tls.get(key) + if value is not None and not _reference(value): + errors.append(f"identity TLS {key} is invalid") + if ( + tls.get("openmetadata_endpoint") is not None + and tls.get("openmetadata_endpoint") == tls.get("gravitino_endpoint") + ): + errors.append("metadata providers must have distinct TLS endpoints") + + catalog = _mapping(profile.get("catalog")) + catalog_keys = { + "decision_status", + "gravitino_backend", + "catalog_reference", + "persistence_reference", + "backup_policy_reference", + "persistent_required", + } + errors.extend(_inventory_errors(catalog, catalog_keys, "identity catalog")) + if catalog.get("decision_status") not in {"pending", "approved"}: + errors.append("identity catalog decision status is invalid") + if catalog.get("gravitino_backend") not in {None, *ALLOWED_CATALOG_BACKENDS}: + errors.append("production Gravitino catalog backend is invalid") + for key in ("catalog_reference", "persistence_reference", "backup_policy_reference"): + value = catalog.get(key) + if value is not None and not _reference(value): + errors.append(f"production catalog {key} is invalid") + if catalog.get("persistent_required") is not True: + errors.append("production Gravitino catalog must require persistence") + + tenancy = _mapping(profile.get("tenancy")) + tenancy_keys = { + "isolation_mode", + "policy_reference", + "tenant_claim", + "cross_tenant_denial_required", + } + errors.extend(_inventory_errors(tenancy, tenancy_keys, "identity tenancy")) + if tenancy.get("isolation_mode") not in {None, "namespace_and_provider_policy"}: + errors.append("identity tenant isolation mode is invalid") + if tenancy.get("policy_reference") is not None and not _reference( + tenancy.get("policy_reference") + ): + errors.append("identity tenant policy reference is invalid") + if ( + tenancy.get("tenant_claim") != "gda_tenant_id" + or tenancy.get("cross_tenant_denial_required") is not True + ): + errors.append("identity tenant contract does not match") + + operations = _mapping(profile.get("operations")) + operation_keys = { + "identity_owner", + "security_owner", + "incident_owner", + "audit_log_reference", + "rotation_slo_minutes", + "revocation_slo_minutes", + "runbook", + "rollback_runbook", + } + errors.extend(_inventory_errors(operations, operation_keys, "identity operations")) + for key in ("identity_owner", "security_owner", "incident_owner", "audit_log_reference"): + value = operations.get(key) + if value is not None and not _reference(value): + errors.append(f"identity operations {key} is invalid") + for key, maximum in (("rotation_slo_minutes", 60), ("revocation_slo_minutes", 15)): + value = operations.get(key) + if value is not None and ( + not isinstance(value, int) + or isinstance(value, bool) + or not 1 <= value <= maximum + ): + errors.append(f"identity operations {key} is invalid") + for name in ("runbook", "rollback_runbook"): + runbook = _mapping(operations.get(name)) + errors.extend(_inventory_errors(runbook, {"uri", "version"}, name)) + if runbook.get("uri") is not None and not _production_https_url( + runbook.get("uri") + ): + errors.append(f"identity {name} URI is invalid") + if runbook.get("version") is not None and _placeholder(runbook.get("version")): + errors.append(f"identity {name} version is invalid") + + claims = _mapping(profile.get("claims")) + errors.extend(_inventory_errors(claims, EXPECTED_CLAIMS, "identity claim")) + for claim in sorted(EXPECTED_CLAIMS): + if claims.get(claim) is not False: + errors.append(f"identity profile may not self-assert production claim: {claim}") + return errors + + +def _profile_blockers(profile: Mapping[str, Any]) -> list[str]: + blockers: list[str] = [] + federation = _mapping(profile.get("federation")) + if federation.get("decision_status") != "approved": + blockers.append("federation.decision_status") + for key in ( + "issuer", + "discovery_uri", + "jwks_uri", + "audience", + "token_exchange_mode", + "trust_policy_reference", + ): + if federation.get(key) is None: + blockers.append(f"federation.{key}") + + providers = _mapping(profile.get("providers")) + for provider in EXPECTED_PROVIDERS: + item = _mapping(providers.get(provider)) + for key in ( + "integration_mode", + "authentication_component_reference", + "environment_binding", + "workload_identity_reference", + "kubernetes_service_account", + "namespace_template", + ): + if item.get(key) is None: + blockers.append(f"providers.{provider}.{key}") + + tls = _mapping(profile.get("tls")) + for key in ( + "openmetadata_endpoint", + "gravitino_endpoint", + "trust_bundle_reference", + "certificate_policy_reference", + ): + if tls.get(key) is None: + blockers.append(f"tls.{key}") + + catalog = _mapping(profile.get("catalog")) + if catalog.get("decision_status") != "approved": + blockers.append("catalog.decision_status") + for key in ( + "gravitino_backend", + "catalog_reference", + "persistence_reference", + "backup_policy_reference", + ): + if catalog.get(key) is None: + blockers.append(f"catalog.{key}") + + tenancy = _mapping(profile.get("tenancy")) + for key in ("isolation_mode", "policy_reference"): + if tenancy.get(key) is None: + blockers.append(f"tenancy.{key}") + + operations = _mapping(profile.get("operations")) + for key in ( + "identity_owner", + "security_owner", + "incident_owner", + "audit_log_reference", + "rotation_slo_minutes", + "revocation_slo_minutes", + ): + if operations.get(key) is None: + blockers.append(f"operations.{key}") + for name in ("runbook", "rollback_runbook"): + runbook = _mapping(operations.get(name)) + for key in ("uri", "version"): + if runbook.get(key) is None: + blockers.append(f"operations.{name}.{key}") + return blockers + + +def _binding_fingerprints(profile: Mapping[str, Any]) -> dict[str, str]: + return { + "federation_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("federation")) + ), + "provider_bindings_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("providers")) + ), + "authorization_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("authorization")) + ), + "tls_fingerprint": recovery._canonical_sha256(_mapping(profile.get("tls"))), + "catalog_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("catalog")) + ), + "tenancy_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("tenancy")) + ), + } + + +def _attestation_errors( + attestation: Mapping[str, Any] | None, + *, + profile: Mapping[str, Any], + profile_fingerprint: str, + bindings: Mapping[str, str], + now: datetime, + max_age: timedelta, +) -> list[str]: + if attestation is None: + return ["production identity attestation is missing"] + errors: list[str] = [] + if _sensitive_paths(attestation): + errors.append("identity attestation contains credential-bearing fields") + expected_inventory = { + "schema", + "environment", + "profile_fingerprint", + "source_revision", + "observed_at", + "expires_at", + "protected_environment", + "verifier_identity", + "evidence_uri", + "provider_versions", + "local_evidence_fingerprints", + *bindings.keys(), + "runbook_versions", + "checks", + } + errors.extend( + _inventory_errors(attestation, expected_inventory, "identity attestation") + ) + if ( + attestation.get("schema") != ATTESTATION_SCHEMA + or attestation.get("environment") != ENVIRONMENT + ): + errors.append("identity attestation schema or environment does not match") + if attestation.get("profile_fingerprint") != profile_fingerprint: + errors.append("identity attestation is not bound to the current profile") + if not SHA40_PATTERN.fullmatch(str(attestation.get("source_revision") or "")): + errors.append("identity attestation source revision is invalid") + if attestation.get("protected_environment") != ENVIRONMENT: + errors.append("identity attestation did not run in protected production") + if not _reference(attestation.get("verifier_identity")): + errors.append("identity attestation verifier identity is invalid") + if not _production_https_url(attestation.get("evidence_uri")): + errors.append("identity attestation evidence URI is invalid") + + try: + observed_at = datetime.fromisoformat(str(attestation.get("observed_at"))) + expires_at = datetime.fromisoformat(str(attestation.get("expires_at"))) + if observed_at.tzinfo is None or observed_at.utcoffset() is None: + raise ValueError + if expires_at.tzinfo is None or expires_at.utcoffset() is None: + raise ValueError + age = now - observed_at + if age < timedelta(seconds=-30) or age > max_age: + errors.append("identity attestation is outside the accepted freshness window") + if expires_at <= now or expires_at <= observed_at: + errors.append("identity attestation has expired or has invalid expiry") + if expires_at - observed_at > timedelta(days=7): + errors.append("identity attestation validity exceeds seven days") + except ValueError: + errors.append("identity attestation timestamps are invalid") + + if _mapping(attestation.get("provider_versions")) != EXPECTED_PROVIDERS: + errors.append("identity attestation provider versions do not match") + local_fingerprints = { + provider: item["evidence_fingerprint"] + for provider, item in EXPECTED_LOCAL_EVIDENCE.items() + } + if _mapping(attestation.get("local_evidence_fingerprints")) != local_fingerprints: + errors.append("identity attestation local evidence bindings do not match") + for key, fingerprint in bindings.items(): + if attestation.get(key) != fingerprint: + errors.append(f"identity attestation binding does not match: {key}") + operations = _mapping(profile.get("operations")) + expected_runbooks = { + name: _mapping(operations.get(name)).get("version") + for name in ("runbook", "rollback_runbook") + } + if _mapping(attestation.get("runbook_versions")) != expected_runbooks: + errors.append("identity attestation runbook versions do not match") + checks = _mapping(attestation.get("checks")) + if set(checks) != EXPECTED_ATTESTATION_CHECKS: + errors.append("identity attestation check inventory does not match") + for check in sorted(EXPECTED_ATTESTATION_CHECKS): + if checks.get(check) != "passed": + errors.append(f"identity attestation check did not pass: {check}") + return errors + + +def _invalid_report(error: str) -> dict[str, Any]: + stable: dict[str, Any] = { + "schema": REPORT_SCHEMA, + "environment": ENVIRONMENT, + "profile_fingerprint": None, + "attestation_fingerprint": None, + "federation_fingerprint": None, + "provider_bindings_fingerprint": None, + "authorization_fingerprint": None, + "tls_fingerprint": None, + "catalog_fingerprint": None, + "tenancy_fingerprint": None, + "profile_valid": False, + "profile_errors": [error], + "profile_blockers": [], + "ready_for_protected_verification": False, + "attestation_valid": False, + "attestation_errors": ["production identity attestation is missing"], + **{claim: False for claim in REPORT_CLAIMS}, + "production_ready": False, + } + return {**stable, "report_fingerprint": recovery._canonical_sha256(stable)} + + +def build_identity_readiness_report( + *, + profile_path: Path | None = None, + attestation: Mapping[str, Any] | None = None, + now: datetime | None = None, + max_attestation_age: timedelta = timedelta(hours=24), +) -> dict[str, Any]: + """Build a deterministic identity readiness report.""" + current = now or datetime.now(UTC) + if current.tzinfo is None or current.utcoffset() is None: + raise MetadataFabricIdentityGateError( + "identity readiness verification time must be timezone-aware" + ) + if max_attestation_age <= timedelta(0): + raise MetadataFabricIdentityGateError( + "identity attestation freshness window must be positive" + ) + path = (profile_path or DEFAULT_PROFILE_PATH).resolve() + try: + profile = _load_yaml_object(path) + except (OSError, TypeError, yaml.YAMLError) as exc: + return _invalid_report(f"identity profile is invalid: {type(exc).__name__}") + + profile_fingerprint = recovery._canonical_sha256(profile) + bindings = _binding_fingerprints(profile) + profile_errors = _profile_errors(profile) + profile_blockers = _profile_blockers(profile) + profile_valid = not profile_errors + ready_for_verification = profile_valid and not profile_blockers + attestation_errors = _attestation_errors( + attestation, + profile=profile, + profile_fingerprint=profile_fingerprint, + bindings=bindings, + now=current, + max_age=max_attestation_age, + ) + attestation_valid = ready_for_verification and not attestation_errors + gate_passed = ready_for_verification and attestation_valid + stable = { + "schema": REPORT_SCHEMA, + "environment": ENVIRONMENT, + "profile_fingerprint": profile_fingerprint, + "attestation_fingerprint": ( + recovery._canonical_sha256(attestation) if attestation is not None else None + ), + **bindings, + "profile_valid": profile_valid, + "profile_errors": profile_errors, + "profile_blockers": profile_blockers, + "ready_for_protected_verification": ready_for_verification, + "attestation_valid": attestation_valid, + "attestation_errors": attestation_errors, + **{claim: gate_passed for claim in REPORT_CLAIMS}, + "production_ready": False, + } + return {**stable, "report_fingerprint": recovery._canonical_sha256(stable)} + + +def verify_report_integrity(report: Mapping[str, Any]) -> list[str]: + """Reject modified identity reports and overall production overclaims.""" + errors: list[str] = [] + if _sensitive_paths(report): + errors.append("identity readiness report contains credential-bearing fields") + errors.extend(_inventory_errors(report, REPORT_INVENTORY, "identity readiness report")) + if report.get("schema") != REPORT_SCHEMA or report.get("environment") != ENVIRONMENT: + errors.append("identity readiness report schema or environment does not match") + stable = {key: value for key, value in report.items() if key != "report_fingerprint"} + if report.get("report_fingerprint") != recovery._canonical_sha256(stable): + errors.append("identity readiness report fingerprint does not match") + if report.get("production_ready") is not False: + errors.append("identity gate may not claim overall production readiness") + expected_gate = ( + report.get("profile_valid") is True + and report.get("profile_blockers") == [] + and report.get("attestation_valid") is True + ) + for claim in sorted(REPORT_CLAIMS): + if report.get(claim) is not expected_gate: + errors.append(f"identity gate result is inconsistent: {claim}") + return errors + + +def _write_report(report: Mapping[str, Any], output: Path | None) -> None: + rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if output is None: + print(rendered, end="") + return + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + evaluate = subparsers.add_parser("evaluate") + evaluate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + evaluate.add_argument("--attestation", type=Path, required=True) + evaluate.add_argument("--output", type=Path) + verify = subparsers.add_parser("verify") + verify.add_argument("--input", type=Path, required=True) + args = parser.parse_args(argv) + try: + if args.command == "validate": + report = build_identity_readiness_report(profile_path=args.profile) + _write_report(report, None) + return 0 if report["profile_valid"] else 1 + if args.command == "evaluate": + attestation = _load_json_object(args.attestation) + report = build_identity_readiness_report( + profile_path=args.profile, + attestation=attestation, + ) + _write_report(report, args.output) + return 0 if report["production_identity_gate_passed"] else 1 + report = _load_json_object(args.input) + errors = verify_report_integrity(report) + _write_report({"verified": not errors, "errors": errors}, None) + return 0 if not errors else 1 + except ( + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + yaml.YAMLError, + MetadataFabricIdentityGateError, + ) as exc: + print(f"metadata identity gate: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/platform_truth.py b/data_agent/platform_truth.py index 303e840a..013c5488 100644 --- a/data_agent/platform_truth.py +++ b/data_agent/platform_truth.py @@ -628,6 +628,26 @@ def _config( ), "Protected OIDC workload identity and authenticated Gravitino access control", ), + RuntimeSpec( + "metadata_gravitino_identity_rehearsal", + "gravitino_identity_rehearsal", + "governed", + "evidence_durable", + "committed local Gravitino Basic bounded identity evidence", + "metadata-platform", + "local_verification_only", + ( + "data_agent/metadata_fabric_gravitino_identity.py", + "scripts/metadata-fabric-gravitino-identity.sh", + ), + ( + ( + "data_agent/metadata_fabric_gravitino_identity.py", + "subprocess.run", + ), + ), + "Protected OIDC workload identity, TLS and production Gravitino catalog", + ), RuntimeSpec( "datalake_monitor", "monitor_loop", diff --git a/data_agent/test_metadata_fabric_gravitino_identity.py b/data_agent/test_metadata_fabric_gravitino_identity.py new file mode 100644 index 00000000..7db34227 --- /dev/null +++ b/data_agent/test_metadata_fabric_gravitino_identity.py @@ -0,0 +1,392 @@ +import base64 +import json +from copy import deepcopy +from pathlib import Path +from urllib.parse import unquote + +import httpx +import yaml +from pydantic import SecretStr + +from data_agent import metadata_fabric_gravitino_identity as identity + + +def _provider_result() -> dict: + return { + "version": "1.3.0", + "configuration": { + "authenticator": "basic", + "access_control_enabled": True, + "idp_extension": "org.apache.gravitino.idp.web.rest.feature", + "built_in_idp_scope": "local_poc_only", + "simple_authenticator_trusted": False, + }, + "authentication": { + "service_admin_status": 200, + "unregistered_principal_status": 401, + "bounded_user_status": 200, + "material_recorded": False, + }, + "principal": { + "name": "gda-metadata-projection", + "granted_name": "gda-metadata-projection", + "roles": ["gda-table-projection"], + "is_service_admin": False, + }, + "role": { + "name": "gda-table-projection", + "readback_name": "gda-table-projection", + "securable_objects": identity._expected_securable_objects(), + }, + "bootstrap": { + "metalake": "gda_identity", + "catalog": "lakehouse", + "schema": "published", + "service_admin_used": True, + "minimum_privilege": False, + }, + "allowed_probe": { + "operation": "CREATE_TABLE", + "resource": "SCHEMA", + "create_status": 200, + "read_status": 200, + "table": "gda_identity_probe", + "readback_table": "gda_identity_probe", + "rotated_readback_table": "gda_identity_probe", + "full_name": "lakehouse.published.gda_identity_probe", + }, + "denied_probe": { + "operation": "CREATE_CATALOG", + "resource": "METALAKE", + "catalog": "unauthorized_catalog", + "status": 403, + }, + "login_lifecycle": { + "rotation_mode": "administrator_reset", + "old_after_rotation_status": 401, + "rotated_authentication_status": 200, + "revocation_mode": "idp_user_delete", + "after_revocation_status": 401, + "idp_principal_absent": True, + "material_recorded": False, + }, + "idp_create_response_present": True, + } + + +def _observation() -> dict: + return { + "schema": identity.OBSERVATION_SCHEMA, + "observed_at": "2026-07-28T14:00:00+00:00", + "contract": { + "contract_fingerprint": "a" * 64, + "local_static_contract_verified": True, + }, + "runtime": { + "context": identity.CONTEXT, + "namespace": { + "name": identity.REHEARSAL_NAMESPACE, + "uid": "11111111-1111-4111-8111-111111111111", + }, + "service": { + "name": "gravitino-identity", + "uid": "22222222-2222-4222-8222-222222222222", + "type": "ClusterIP", + }, + "gravitino": { + "kind": "StatefulSet", + "name": "gravitino-identity", + "uid": "33333333-3333-4333-8333-333333333333", + "image": "gda/gravitino:1.3.0-local-arm64", + "service_account": "gravitino-identity", + "service_account_automount_disabled": True, + "ready_replicas": 1, + }, + "postgresql": { + "kind": "StatefulSet", + "name": "gravitino-identity-postgresql", + "uid": "44444444-4444-4444-8444-444444444444", + "image": "postgres:16.10-bookworm", + "service_account": "gravitino-identity-postgresql", + "service_account_automount_disabled": True, + "ready_replicas": 1, + "ephemeral_data_volume": True, + }, + "source_schema_sha256": identity.GRAVITINO_SCHEMA_SHA256, + }, + "gravitino": _provider_result(), + "runtime_checks": { + "namespace_delete_completed": True, + "namespace_absent": True, + "provider_objects_retained": False, + "all_port_forwards_stopped": True, + "material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }, + } + + +def test_static_contract_is_valid_and_explicitly_local(): + report = identity.build_contract_report() + + assert report["local_static_contract_verified"] is True + assert report["errors"] == [] + assert report["authentication"]["authenticator"] == "basic" + assert report["authentication"]["simple_authenticator_trusted"] is False + assert report["gravitino_authentication_verified"] is False + assert report["protected_workload_identity_verified"] is False + assert report["production_ready"] is False + assert all( + not Path(item["path"]).is_absolute() for item in report["files"].values() + ) + + +def test_profile_rejects_broad_privileges_and_claim_overreach(tmp_path): + profile = yaml.safe_load(identity.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + profile["scope"]["role_securable_objects"][1]["privileges"].append( + {"name": "MODIFY_TABLE", "condition": "ALLOW"} + ) + profile["claims"]["production_ready"] = True + target = tmp_path / "profile.yaml" + target.write_text(yaml.safe_dump(profile), encoding="utf-8") + + report = identity.build_contract_report(profile_path=target) + + assert report["local_static_contract_verified"] is False + assert "profile is invalid" in "\n".join(report["errors"]) + + +def test_profile_rejects_inline_sensitive_material(tmp_path): + profile = yaml.safe_load(identity.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + profile["identity"]["password"] = "must-not-enter-config" + target = tmp_path / "profile.yaml" + target.write_text(yaml.safe_dump(profile), encoding="utf-8") + + report = identity.build_contract_report(profile_path=target) + + assert report["local_static_contract_verified"] is False + + +def test_valid_evidence_proves_only_local_gravitino_basic_identity(): + evidence = identity.build_evidence(_observation()) + + assert evidence["status"] == "local_gravitino_basic_identity_verified" + assert evidence["local_gravitino_basic_identity_verified"] is True + assert evidence["local_gravitino_minimum_privilege_verified"] is True + assert evidence["local_gravitino_login_rotation_verified"] is True + assert evidence["local_gravitino_revocation_verified"] is True + assert evidence["gravitino_authentication_verified"] is False + assert evidence["protected_workload_identity_verified"] is False + assert evidence["oidc_verified"] is False + assert evidence["production_ready"] is False + assert identity.verify_evidence_integrity(evidence) == [] + + +def test_evidence_blocks_broad_role_failed_denial_and_incomplete_cleanup(): + observation = _observation() + observation["gravitino"]["role"]["securable_objects"][1][ + "privileges" + ].append({"name": "MODIFY_TABLE", "condition": "ALLOW"}) + observation["gravitino"]["denied_probe"]["status"] = 200 + observation["runtime_checks"]["namespace_absent"] = False + + evidence = identity.build_evidence(observation) + + assert evidence["status"] == "blocked" + rendered = "\n".join(evidence["errors"]) + assert "role exceeds" in rendered + assert "catalog-create denial" in rendered + assert "cleanup is incomplete" in rendered + + +def test_evidence_blocks_login_lifecycle_drift_and_sensitive_fields(): + observation = _observation() + observation["gravitino"]["login_lifecycle"][ + "old_after_rotation_status" + ] = 200 + observation["gravitino"]["api_token"] = "must-not-enter-evidence" + + evidence = identity.build_evidence(observation) + + assert evidence["status"] == "blocked" + rendered = "\n".join(evidence["errors"]) + assert "sensitive material" in rendered + assert "rotation or revocation" in rendered + + +def test_evidence_integrity_rejects_claim_and_observation_tampering(): + evidence = identity.build_evidence(_observation()) + tampered = deepcopy(evidence) + tampered["gravitino_authentication_verified"] = True + tampered["observation"]["gravitino"]["denied_probe"]["status"] = 200 + + errors = identity.verify_evidence_integrity(tampered) + + assert any("content or fingerprint drifted" in error for error in errors) + assert any("gravitino_authentication_verified" in error for error in errors) + + +class _GravitinoMock: + def __init__(self) -> None: + self.admin_name = "gda-identity-admin" + self.admin_value = "admin-local-material" + self.user_name = "gda-metadata-projection" + self.user_value: str | None = None + self.user_roles: list[str] = [] + self.role: dict | None = None + self.table: dict | None = None + + @staticmethod + def _response(status: int, payload: dict | None = None) -> httpx.Response: + return ( + httpx.Response(status, json=payload) + if payload is not None + else httpx.Response(status) + ) + + @staticmethod + def _login(request: httpx.Request) -> tuple[str, str] | None: + value = request.headers.get("Authorization", "") + if not value.startswith("Basic "): + return None + decoded = base64.b64decode(value.removeprefix("Basic ")).decode("utf-8") + name, material = decoded.split(":", 1) + return name, material + + def __call__(self, request: httpx.Request) -> httpx.Response: + path = request.url.path.removeprefix("/api/") + method = request.method + body = json.loads(request.content) if request.content else {} + login = self._login(request) + is_admin = login == (self.admin_name, self.admin_value) + is_user = login == (self.user_name, self.user_value) and self.user_value is not None + + if path == "version" and method == "GET": + if not (is_admin or is_user): + return self._response(401, {"code": 1001}) + return self._response(200, {"code": 0, "version": {"version": "1.3.0"}}) + + if not is_admin and not is_user: + return self._response(401, {"code": 1001}) + + if path == "metalakes" and method == "POST" and is_admin: + return self._response(200, {"code": 0, "metalake": body}) + if path == "metalakes/gda_identity/catalogs" and method == "POST": + if is_user: + return self._response(403, {"code": 1002}) + return self._response(200, {"code": 0, "catalog": body}) + if ( + path == "metalakes/gda_identity/catalogs/lakehouse/schemas" + and method == "POST" + and is_admin + ): + return self._response(200, {"code": 0, "schema": body}) + if path == "idp/users" and method == "POST" and is_admin: + self.user_value = body["password"] + return self._response( + 200, {"code": 0, "user": {"name": body["user"]}} + ) + if ( + path == "metalakes/gda_identity/users" + and method == "POST" + and is_admin + ): + return self._response( + 200, {"code": 0, "user": {"name": body["name"], "roles": []}} + ) + if ( + path == "metalakes/gda_identity/roles" + and method == "POST" + and is_admin + ): + self.role = body + return self._response(200, {"code": 0, "role": body}) + if ( + path + == "metalakes/gda_identity/permissions/users/gda-metadata-projection/grant" + and method == "PUT" + and is_admin + ): + self.user_roles = body["roleNames"] + return self._response( + 200, + { + "code": 0, + "user": {"name": self.user_name, "roles": self.user_roles}, + }, + ) + if ( + path == "metalakes/gda_identity/roles/gda-table-projection" + and method == "GET" + and is_admin + ): + role = deepcopy(self.role) + for securable_object in role["securableObjects"]: + securable_object["type"] = securable_object["type"].lower() + for privilege in securable_object["privileges"]: + privilege["name"] = privilege["name"].lower() + privilege["condition"] = privilege["condition"].lower() + return self._response(200, {"code": 0, "role": role}) + if ( + path == "metalakes/gda_identity/catalogs/lakehouse/schemas/published/tables" + and method == "POST" + and is_user + ): + self.table = body + return self._response(200, {"code": 0, "table": body}) + if ( + path + == "metalakes/gda_identity/catalogs/lakehouse/schemas/published/tables/gda_identity_probe" + and method == "GET" + and is_user + ): + return self._response(200, {"code": 0, "table": self.table}) + if path == "idp/users/gda-metadata-projection" and method == "PUT" and is_admin: + self.user_value = body["password"] + return self._response(200, {"code": 0, "user": {"name": self.user_name}}) + if ( + path == "idp/users/gda-metadata-projection" + and method == "DELETE" + and is_admin + ): + self.user_value = None + return self._response(200, {"code": 0, "dropped": True}) + if ( + path == "idp/users/gda-metadata-projection" + and method == "GET" + and is_admin + ): + return self._response(404, {"code": 1003}) + return self._response(404, {"code": 1003, "path": unquote(path)}) + + +def test_http_rehearsal_enforces_basic_role_rotation_revocation_and_denial(): + profile = identity.load_profile() + provider = _GravitinoMock() + rehearsal = identity.GravitinoIdentityRehearsal( + base_url="http://gravitino.test/api", + admin_name=provider.admin_name, + admin_material=SecretStr(provider.admin_value), + transport=httpx.MockTransport(provider), + ) + + try: + result = rehearsal.execute( + profile, + initial_material=SecretStr("initial-local-material"), + rotated_material=SecretStr("rotated-local-material"), + ) + finally: + rehearsal.close() + + assert result["authentication"]["unregistered_principal_status"] == 401 + assert result["allowed_probe"]["create_status"] == 200 + assert result["denied_probe"]["status"] == 403 + assert result["login_lifecycle"]["old_after_rotation_status"] == 401 + assert result["login_lifecycle"]["rotated_authentication_status"] == 200 + assert result["login_lifecycle"]["after_revocation_status"] == 401 + assert result["login_lifecycle"]["idp_principal_absent"] is True + rendered = json.dumps(result) + assert "initial-local-material" not in rendered + assert "rotated-local-material" not in rendered + assert provider.admin_value not in rendered diff --git a/data_agent/test_metadata_fabric_identity_gate.py b/data_agent/test_metadata_fabric_identity_gate.py new file mode 100644 index 00000000..f726059b --- /dev/null +++ b/data_agent/test_metadata_fabric_identity_gate.py @@ -0,0 +1,394 @@ +import json +from copy import deepcopy +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +import yaml + +from data_agent import metadata_fabric_identity_gate as gate + + +NOW = datetime(2026, 7, 28, 16, 0, tzinfo=UTC) + + +def _default_profile() -> dict: + return yaml.safe_load(gate.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + + +def _write_profile(tmp_path: Path, profile: dict) -> Path: + target = tmp_path / "identity-profile.yaml" + target.write_text(yaml.safe_dump(profile, sort_keys=False), encoding="utf-8") + return target + + +def _complete_profile() -> dict: + profile = _default_profile() + profile["federation"].update( + { + "decision_status": "approved", + "issuer": "https://identity.gda.internal", + "discovery_uri": ( + "https://identity.gda.internal/.well-known/openid-configuration" + ), + "jwks_uri": "https://identity.gda.internal/oauth2/jwks", + "audience": "audience://metadata-fabric/production", + "token_exchange_mode": "oidc_workload_federation", + "trust_policy_reference": "policy://production/metadata-federation-v1", + } + ) + for provider, mode in ( + ("openmetadata", "provider_native_oidc"), + ("gravitino", "custom_oidc_authenticator"), + ): + item = profile["providers"][provider] + item.update( + { + "integration_mode": mode, + "environment_binding": f"environment://production/{provider}", + "workload_identity_reference": ( + f"identity://production/metadata/{provider}" + ), + "kubernetes_service_account": provider, + "namespace_template": "gda-{tenant_id}", + } + ) + digest_character = "a" if provider == "openmetadata" else "b" + item["authentication_component_reference"] = ( + f"oci://registry.gda.internal/metadata/{provider}-identity@sha256:" + f"{digest_character * 64}" + ) + profile["tls"].update( + { + "minimum_version": "TLSv1.3", + "openmetadata_endpoint": "https://openmetadata.gda.internal/api", + "gravitino_endpoint": "https://gravitino.gda.internal/api", + "trust_bundle_reference": "pki://production/metadata-fabric-ca-v1", + "certificate_policy_reference": ( + "policy://production/metadata-certificate-v1" + ), + } + ) + profile["catalog"].update( + { + "decision_status": "approved", + "gravitino_backend": "iceberg_rest", + "catalog_reference": "catalog://production/lakehouse", + "persistence_reference": "storage://production/metadata-catalog", + "backup_policy_reference": "policy://production/metadata-backup-v1", + } + ) + profile["tenancy"].update( + { + "isolation_mode": "namespace_and_provider_policy", + "policy_reference": "policy://production/metadata-tenant-isolation-v1", + } + ) + profile["operations"].update( + { + "identity_owner": "team://metadata-platform", + "security_owner": "team://security-platform", + "incident_owner": "team://metadata-oncall", + "audit_log_reference": "logging://production/metadata-identity-audit", + "rotation_slo_minutes": 30, + "revocation_slo_minutes": 10, + "runbook": { + "uri": "https://runbooks.gda.internal/metadata-identity", + "version": "2026.07.28", + }, + "rollback_runbook": { + "uri": "https://runbooks.gda.internal/metadata-identity-rollback", + "version": "2026.07.28", + }, + } + ) + return profile + + +def _attestation(profile_path: Path, **changes: object) -> dict: + report = gate.build_identity_readiness_report(profile_path=profile_path, now=NOW) + profile = yaml.safe_load(profile_path.read_text(encoding="utf-8")) + payload = { + "schema": gate.ATTESTATION_SCHEMA, + "environment": gate.ENVIRONMENT, + "profile_fingerprint": report["profile_fingerprint"], + "source_revision": "c" * 40, + "observed_at": NOW.isoformat(), + "expires_at": (NOW + timedelta(days=1)).isoformat(), + "protected_environment": gate.ENVIRONMENT, + "verifier_identity": "identity://github/metadata-identity-verifier", + "evidence_uri": "https://evidence.gda.internal/metadata-identity/run-42", + "provider_versions": deepcopy(gate.EXPECTED_PROVIDERS), + "local_evidence_fingerprints": { + provider: item["evidence_fingerprint"] + for provider, item in gate.EXPECTED_LOCAL_EVIDENCE.items() + }, + "federation_fingerprint": report["federation_fingerprint"], + "provider_bindings_fingerprint": report["provider_bindings_fingerprint"], + "authorization_fingerprint": report["authorization_fingerprint"], + "tls_fingerprint": report["tls_fingerprint"], + "catalog_fingerprint": report["catalog_fingerprint"], + "tenancy_fingerprint": report["tenancy_fingerprint"], + "runbook_versions": { + name: profile["operations"][name]["version"] + for name in ("runbook", "rollback_runbook") + }, + "checks": {name: "passed" for name in gate.EXPECTED_ATTESTATION_CHECKS}, + } + payload.update(changes) + return payload + + +def test_checked_in_profile_is_valid_but_production_identity_is_blocked(): + report = gate.build_identity_readiness_report(now=NOW) + + assert report["profile_valid"] is True + assert report["profile_errors"] == [] + assert report["ready_for_protected_verification"] is False + assert report["production_identity_gate_passed"] is False + assert report["provider_minimum_privilege_verified"] is False + assert report["protected_workload_identity_verified"] is False + assert report["oidc_verified"] is False + assert report["tls_verified"] is False + assert report["production_identity_verified"] is False + assert report["production_ready"] is False + assert len(report["profile_blockers"]) == 40 + assert "federation.decision_status" in report["profile_blockers"] + assert "providers.gravitino.integration_mode" in report["profile_blockers"] + assert "catalog.gravitino_backend" in report["profile_blockers"] + assert gate.verify_report_integrity(report) == [] + + +def test_complete_profile_without_attestation_is_only_ready_for_verification(tmp_path): + profile_path = _write_profile(tmp_path, _complete_profile()) + + report = gate.build_identity_readiness_report(profile_path=profile_path, now=NOW) + + assert report["profile_valid"] is True + assert report["profile_errors"] == [] + assert report["profile_blockers"] == [] + assert report["ready_for_protected_verification"] is True + assert report["attestation_valid"] is False + assert report["production_identity_gate_passed"] is False + + +def test_fresh_bound_attestation_passes_only_the_identity_gate(tmp_path): + profile_path = _write_profile(tmp_path, _complete_profile()) + + report = gate.build_identity_readiness_report( + profile_path=profile_path, + attestation=_attestation(profile_path), + now=NOW, + ) + + assert report["attestation_valid"] is True + assert report["production_identity_gate_passed"] is True + assert report["provider_minimum_privilege_verified"] is True + assert report["protected_workload_identity_verified"] is True + assert report["oidc_verified"] is True + assert report["tls_verified"] is True + assert report["credential_rotation_verified"] is True + assert report["credential_revocation_verified"] is True + assert report["persistent_catalog_identity_binding_verified"] is True + assert report["production_identity_verified"] is True + assert report["production_ready"] is False + assert gate.verify_report_integrity(report) == [] + + +@pytest.mark.parametrize( + ("provider", "mode"), + [ + ("openmetadata", "static_jwt"), + ("gravitino", "simple"), + ("gravitino", "basic"), + ], +) +def test_profile_rejects_local_or_unvalidated_authenticators(tmp_path, provider, mode): + profile = _complete_profile() + profile["providers"][provider]["integration_mode"] = mode + + report = gate.build_identity_readiness_report( + profile_path=_write_profile(tmp_path, profile), now=NOW + ) + + assert report["profile_valid"] is False + assert f"{provider} identity integration mode is invalid" in "\n".join( + report["profile_errors"] + ) + assert report["production_identity_gate_passed"] is False + + +def test_profile_rejects_static_credentials_unpinned_component_and_http(tmp_path): + profile = _complete_profile() + profile["providers"]["gravitino"]["static_credentials_forbidden"] = False + profile["providers"]["openmetadata"]["authentication_component_reference"] = ( + "oci://registry.gda.internal/metadata/openmetadata-identity:latest" + ) + profile["tls"]["gravitino_endpoint"] = "http://localhost:8090/api" + + report = gate.build_identity_readiness_report( + profile_path=_write_profile(tmp_path, profile), now=NOW + ) + + rendered = "\n".join(report["profile_errors"]) + assert report["profile_valid"] is False + assert "static_credentials_forbidden" in rendered + assert "digest-pinned OCI" in rendered + assert "gravitino_endpoint is invalid" in rendered + + +def test_profile_rejects_authorization_and_local_evidence_drift(tmp_path): + profile = _complete_profile() + profile["authorization"]["gravitino"]["securable_objects"][1][ + "privileges" + ].append("MODIFY_TABLE") + profile["scope"]["local_evidence"]["openmetadata"][ + "evidence_fingerprint" + ] = "0" * 64 + + report = gate.build_identity_readiness_report( + profile_path=_write_profile(tmp_path, profile), now=NOW + ) + + rendered = "\n".join(report["profile_errors"]) + assert report["profile_valid"] is False + assert "minimum-privilege authorization contract" in rendered + assert "openmetadata local identity evidence binding" in rendered + + +def test_profile_rejects_sensitive_fields_and_self_asserted_claims(tmp_path): + profile = _complete_profile() + profile["federation"]["client_secret"] = "must-not-enter-profile" + profile["claims"]["production_identity_gate_passed"] = True + + report = gate.build_identity_readiness_report( + profile_path=_write_profile(tmp_path, profile), now=NOW + ) + + rendered = "\n".join(report["profile_errors"]) + assert report["profile_valid"] is False + assert "credential-bearing fields" in rendered + assert "federation inventory" in rendered + assert "may not self-assert" in rendered + + +@pytest.mark.parametrize( + "binding", + [ + "profile_fingerprint", + "federation_fingerprint", + "provider_bindings_fingerprint", + "authorization_fingerprint", + "tls_fingerprint", + "catalog_fingerprint", + "tenancy_fingerprint", + ], +) +def test_attestation_rejects_every_profile_binding_drift(tmp_path, binding): + profile_path = _write_profile(tmp_path, _complete_profile()) + attestation = _attestation(profile_path) + attestation[binding] = "0" * 64 + + report = gate.build_identity_readiness_report( + profile_path=profile_path, attestation=attestation, now=NOW + ) + + assert report["attestation_valid"] is False + assert "current profile" in "\n".join(report["attestation_errors"]) or ( + f"binding does not match: {binding}" in "\n".join(report["attestation_errors"]) + ) + + +def test_attestation_rejects_sensitive_fields_expiry_and_local_evidence_drift(tmp_path): + profile_path = _write_profile(tmp_path, _complete_profile()) + attestation = _attestation( + profile_path, + observed_at=(NOW - timedelta(days=2)).isoformat(), + expires_at=(NOW - timedelta(days=1)).isoformat(), + access_token="must-not-enter-attestation", + ) + attestation["local_evidence_fingerprints"]["gravitino"] = "0" * 64 + + report = gate.build_identity_readiness_report( + profile_path=profile_path, attestation=attestation, now=NOW + ) + + rendered = "\n".join(report["attestation_errors"]) + assert report["attestation_valid"] is False + assert "credential-bearing fields" in rendered + assert "freshness window" in rendered + assert "expired" in rendered + assert "local evidence bindings" in rendered + + +@pytest.mark.parametrize( + "check", + [ + "gravitino_administrative_deny", + "provider_direct_access_denied", + "credential_revocation", + "persistent_catalog_restart", + "cross_tenant_denial", + "rollback_rehearsal", + ], +) +def test_attestation_requires_denial_lifecycle_persistence_and_rollback(tmp_path, check): + profile_path = _write_profile(tmp_path, _complete_profile()) + attestation = _attestation(profile_path) + attestation["checks"][check] = "failed" + + report = gate.build_identity_readiness_report( + profile_path=profile_path, attestation=attestation, now=NOW + ) + + assert report["attestation_valid"] is False + assert check in "\n".join(report["attestation_errors"]) + + +def test_report_integrity_rejects_tampering_inventory_and_production_overclaim(): + report = gate.build_identity_readiness_report(now=NOW) + report["production_ready"] = True + report["oidc_verified"] = True + + errors = gate.verify_report_integrity(report) + + assert "identity readiness report fingerprint does not match" in errors + assert "identity gate may not claim overall production readiness" in errors + assert "identity gate result is inconsistent: oidc_verified" in errors + + forged = gate.build_identity_readiness_report(now=NOW) + forged["unexpected_claim"] = False + stable = { + key: value for key, value in forged.items() if key != "report_fingerprint" + } + forged["report_fingerprint"] = gate.recovery._canonical_sha256(stable) + assert "identity readiness report inventory does not match" in ( + gate.verify_report_integrity(forged) + ) + + +def test_wrapper_is_fail_closed_and_malformed_profile_is_blocked(tmp_path): + wrapper = gate.REPO_ROOT / "scripts/metadata-fabric-identity-gate.sh" + text = wrapper.read_text(encoding="utf-8") + assert "set -euo pipefail" in text + assert "metadata_fabric_identity_gate" in text + + target = tmp_path / "profile.yaml" + target.write_text("providers: [\n", encoding="utf-8") + report = gate.build_identity_readiness_report(profile_path=target, now=NOW) + assert report["profile_valid"] is False + assert report["production_identity_gate_passed"] is False + assert gate.verify_report_integrity(report) == [] + + +def test_attestation_never_records_material_in_success_report(tmp_path): + profile_path = _write_profile(tmp_path, _complete_profile()) + report = gate.build_identity_readiness_report( + profile_path=profile_path, + attestation=_attestation(profile_path), + now=NOW, + ) + rendered = json.dumps(report, sort_keys=True) + assert "client_secret" not in rendered + assert "access_token" not in rendered + assert report["production_identity_gate_passed"] is True diff --git a/data_agent/test_platform_truth.py b/data_agent/test_platform_truth.py index d5625d7a..f07dac43 100644 --- a/data_agent/test_platform_truth.py +++ b/data_agent/test_platform_truth.py @@ -208,6 +208,11 @@ def test_repository_source_access_and_runtime_baselines_match(): and item["production_role"] == "local_verification_only" for item in static_report["runtime"]["inventory"] ) + assert any( + item["runtime_id"] == "metadata_gravitino_identity_rehearsal" + and item["production_role"] == "local_verification_only" + for item in static_report["runtime"]["inventory"] + ) def test_runtime_report_detects_unregistered_background_mechanism(tmp_path): diff --git a/docs/architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md b/docs/architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md new file mode 100644 index 00000000..964d160a --- /dev/null +++ b/docs/architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md @@ -0,0 +1,67 @@ +# ADR-052: Local Gravitino Basic Bounded Provider Identity + +**Status**: Accepted + +**Date**: 2026-07-28 + +**Decision owners**: Metadata Platform, Data Governance, Security, Platform Architecture + +**Related decisions**: [ADR-006](adr-006-openmetadata-governance-and-active-metadata-platform.md) · [ADR-024](adr-024-dispatch-authorization-evidence.md) · [ADR-037](adr-037-local-metadata-fabric-foundation-sandbox.md) · [ADR-047](adr-047-deterministic-metadata-fabric-ingestion-projection.md) · [ADR-051](adr-051-local-openmetadata-bounded-provider-identity.md) + +## Context + +M3-2 used the OpenMetadata bootstrap administrator and an unauthenticated Gravitino `simple` authenticator. M3-5 then proved a bounded OpenMetadata grant, but the Gravitino side still had no honest authentication or authorization evidence. Gravitino `1.3.0` ships a Basic IdP extension, so a separately isolated rehearsal can validate the provider-native identity boundary without mutating the retained M3-2 projection. + +The `simple` authenticator is deliberately excluded: upstream behavior accepts unvalidated usernames and therefore cannot count as authentication. This rehearsal also cannot establish protected Kubernetes workload identity, OIDC federation, TLS, durable catalog conformance or production readiness. + +## Decision + +### 1. Run an isolated, ephemeral provider + +The rehearsal uses Docker Desktop Kubernetes context `docker-desktop` and a pre-existing, checksum-verified Gravitino PostgreSQL schema (`7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df`). It creates only the `gda-metadata-identity` namespace, two single-replica StatefulSets, ClusterIP Services, dedicated ServiceAccounts with token automount disabled, and runtime-generated Kubernetes Secret values. The namespace is deleted before evidence can pass; no Secret value or provider error body enters source, evidence or logs. + +The server is Gravitino `1.3.0` (`gda/gravitino:1.3.0-local-arm64`) with `gravitino.authenticators=basic`, the Basic IdP REST extension and `gravitino.authorization.enable=true`. The provider's relational entity store uses the ephemeral PostgreSQL instance; the probe catalog remains an ephemeral memory catalog and is not a production technical-metadata authority. + +### 2. Keep the administrator as a provisioner only + +The built-in Basic IdP administrator `gda-identity-admin` creates the temporary metalake, catalog, schema, user, role and grant. The exercised principal `gda-metadata-projection` is not a service administrator and receives exactly `gda-table-projection`. + +The role contains only: + +- `USE_CATALOG` on catalog `lakehouse`; +- `USE_SCHEMA` and `CREATE_TABLE` on schema `lakehouse.published`. + +The API client sends the Gravitino 1.3.0 `RoleGrantRequest` field `roleNames` and normalizes provider read-back enum casing at the HTTP boundary. Any extra privilege, role, object or changed scope blocks evidence. + +### 3. Require positive, negative and lifecycle probes + +The rehearsal must observe service-admin authentication (200), an unregistered principal rejection (401), bounded-user authentication (200), table create/read (200/200), and catalog create denial (403). The administrator resets the user password; the old material must return 401 and the replacement must read the probe table. The administrator then deletes the IdP user; the replacement material must return 401 and the IdP lookup must be absent. + +The four local claims are intentionally narrow: `local_gravitino_basic_identity_verified`, `local_gravitino_minimum_privilege_verified`, `local_gravitino_login_rotation_verified` and `local_gravitino_revocation_verified`. They do not imply a protected workload identity or a production provider identity. + +### 4. Preserve the claim boundary + +Evidence fixes `gravitino_authentication_verified=false`, `provider_minimum_privilege_verified=false`, `protected_workload_identity_verified=false`, `oidc_verified=false`, `tls_verified=false`, `production_identity_verified=false` and `production_ready=false`. The local provider result is not combined with M3-5 into a dual-provider production claim: M3-2 still uses the bootstrap administrator, and this rehearsal is isolated from the retained projection and binding ledger. + +## Verification + +The final Docker Desktop rehearsal on 2026-07-28 produced evidence fingerprint `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd` and contract fingerprint `c1c8b08e52c88742fd273378735491626bc4a185f3416746eafa4932374a3704`. + +- Gravitino `1.3.0` and PostgreSQL each reached one ready replica; +- the Basic IdP rejected an unregistered principal with 401 and authenticated the bounded user with 200; +- the bounded role read back exactly `USE_CATALOG`, `USE_SCHEMA` and `CREATE_TABLE` on the declared catalog/schema; +- table create/read returned 200/200 and unauthorized catalog create returned 403; +- password reset invalidated the old material (401), the replacement read the table (200), and IdP deletion invalidated the replacement (401); +- namespace deletion completed, the namespace was absent, all port-forwards stopped and no provider object was retained. + +Focused tests cover strict profile and manifest validation, sensitive-field rejection, exact role scope, positive/negative probes, enum read-back normalization, rotation/revocation, cleanup and evidence tampering. Required CI validates the static contract and runs the focused test module; live Kubernetes evidence remains a local verification artifact. + +## Consequences + +**Positive**: Gravitino now has a real, bounded local authentication and authorization rehearsal with explicit lifecycle and cleanup evidence. The API contract is checked against the actual 1.3.0 DTO instead of relying on a mock-only request shape. + +**Negative**: Basic IdP credentials are locally provisioned by an administrator, the catalog is ephemeral, transport is loopback HTTP, and the provider workload is not authenticated with its Kubernetes ServiceAccount. + +**Mitigation**: retain all production and protected-identity claims as false. The next identity gate must select and validate the protected OIDC/workload exchange, Secret delivery and rotation mechanism for both metadata providers, durable catalog profile, TLS, tenant policy and production conformance. + +**Revisit trigger**: Gravitino version, Basic IdP API, authorization model, catalog backend, cluster identity model or production deployment profile changes. diff --git a/docs/architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md b/docs/architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md new file mode 100644 index 00000000..5165840e --- /dev/null +++ b/docs/architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md @@ -0,0 +1,93 @@ +# ADR-053: Production Metadata Fabric Identity Readiness Gate + +**Status**: Accepted + +**Date**: 2026-07-28 + +**Decision owners**: Metadata Platform, Security, SRE, Platform Architecture + +**Related decisions**: [ADR-019](adr-019-configuration-and-runtime-truth.md) · [ADR-046](adr-046-production-network-policy-readiness-gate.md) · [ADR-051](adr-051-local-openmetadata-bounded-provider-identity.md) · [ADR-052](adr-052-local-gravitino-basic-bounded-provider-identity.md) + +## Context + +M3-5 与 M3-6 已分别证明 OpenMetadata `1.13.1` 临时 bot 和 Gravitino `1.3.0` Basic user 的本地最小权限、越权拒绝、credential rotation/revocation 与完整清理。两次演练都依赖本地管理员 provisioner、loopback HTTP 和短生命周期凭据；Gravitino probe 还使用 memory catalog。因此它们不能证明双 provider 的生产 OIDC、受保护 workload identity、TLS、持久 catalog、tenant isolation 或生产 credential delivery。 + +对 Gravitino `1.3.0` 镜像的检查只发现 `/opt/gravitino/libs/gravitino-idp-basic-1.3.0.jar`，没有发现随镜像交付的原生 OIDC extension。生产设计不能假设不存在的 native OIDC 能力，也不能把 Basic IdP 包装成生产联邦身份。本切片需要把生产身份决策和验收条件冻结为机器可验证、默认关闭的合同，但不替团队选择 IdP、部署认证组件或制造生产证明。 + +## Options Considered + +| 方案 | 优点 | 代价/风险 | 结论 | +|---|---|---|---| +| 将 M3-5/M3-6 本地结果直接提升为生产身份 | 无新增实现 | 本地 JWT/Basic、HTTP 与 memory catalog 没有生产身份语义 | 拒绝 | +| 假设 Gravitino 内置 native OIDC | 配置看似简单 | 与 `1.3.0` 镜像内容不符，形成不可执行合同 | 拒绝 | +| 继续只在 roadmap 列举 IAM 缺口 | 无新增代码 | CI 无法拒绝 static credential、placeholder、过期证明或配置漂移 | 拒绝 | +| 版本化 pending profile + 独立、新鲜且完全绑定的 attestation | 决策与运行证据分离；可在未选型时诚实 fail closed | 需要后续 owner 明确选型并维护证明生命周期 | 采用，限定为 M3-7 | + +## Decision + +### 1. Checked-in profile 固定生产身份边界，不代表部署 + +`config/metadata-fabric-identity.production.yaml` 固定： + +- OpenMetadata `1.13.1`、Gravitino `1.3.0` 及 M3-5/M3-6 两份本地 evidence fingerprint； +- OIDC federation、`sub` workload subject、`gda_tenant_id` tenant claim、最长 900 秒 token 与 short-lived token exchange； +- 每个 provider 的 integration mode、digest-pinned authentication component、环境和 Kubernetes ServiceAccount binding，以及禁止绕过受证明身份路径； +- M3-5/M3-6 已验证的精确 allow/deny authorization contract； +- TLS 1.2+、内部 mTLS、持久 Gravitino catalog、namespace + provider policy tenant isolation； +- identity/security/incident owner、audit、rotation/revocation SLO、runbook 与 rollback runbook； +- 所有 self-reported production claim 固定为 `false`。 + +OpenMetadata 只允许 `provider_native_oidc` 或 `identity_aware_proxy`；Gravitino 只允许 `custom_oidc_authenticator` 或 `identity_aware_proxy`。`simple`、本地 `basic`、静态 JWT、密码或长生命周期 credential 都不能通过该门禁。认证组件必须是 digest-pinned OCI reference。 + +`null` 和 `decision_status=pending` 是合法的显式 blockers，所以当前 profile 可以结构有效而不假装外部决策已完成。placeholder、loopback/cluster-local 或非 HTTPS endpoint、credential-bearing 字段、mutable OCI reference、扩大后的 provider permission、provider version/local evidence 漂移或自报生产结论都会使 profile 无效。 + +### 2. 生产结论只能从受保护 attestation 派生 + +`data_agent.metadata_fabric_identity_gate` 将结果分为三层： + +1. `profile_valid`：profile 的结构、安全边界和本地 evidence binding 可信； +2. `ready_for_protected_verification`：federation、双 provider、TLS、catalog、tenancy 和 operations 的 40 项外部输入已经明确； +3. `production_identity_gate_passed`：另有新鲜的 production protected-environment attestation，且绑定当前 profile、source revision、provider versions、本地 evidence、六组派生 fingerprint 和两个 runbook version。 + +attestation 的精确 18 项检查必须全部为 `passed`：OIDC discovery/token exchange、workload subject 和 tenant claim binding、双 provider allow/administrative deny、provider direct-access bypass denial、static credential absence、rotation/revocation、TLS/mTLS、持久 catalog restart、cross-tenant denial、audit delivery 和 rollback rehearsal。观测时间不得早于验证时刻 24 小时，expiry 必须在未来且有效期最长七天，evidence URI 必须是非本地 HTTPS；任何漂移、过期或失败都关闭门禁。 + +### 3. 身份门禁通过也不等于平台生产就绪 + +同一有效 attestation 可唯一派生 provider minimum privilege、protected workload identity、OIDC、TLS、rotation/revocation、persistent catalog identity binding 和 production identity claims。报告中的 `production_ready` 始终固定为 `false`；生产 recovery、observability、NetworkPolicy、upgrade、registry provenance、持久 binding、ingestion/OpenLineage conformance 等退出门仍须独立通过。 + +`validate` 只验证 checked-in profile，因此有效的 pending contract 在 CI 中成功；`evaluate` 必须提供 attestation，且仅在身份门禁实际通过时成功；`verify` 拒绝 fingerprint 漂移、派生 claim 不一致和 overall production overclaim。本切片不提交真实 attestation、不部署认证组件，也不修改 provider。 + +## Verification + +当前 checked-in profile： + +- profile fingerprint：`2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`； +- report fingerprint：`c607589ee25a87acc8a1ab71372618a9a4c10c1e8ebff15b8db7e78b37600b9f`； +- `profile_valid=true` 且无 profile errors； +- 40 项 federation/provider/TLS/catalog/tenancy/operations 外部输入以 blockers 暴露； +- `ready_for_protected_verification=false`、`attestation_valid=false`； +- 全部生产身份 claims 与 `production_ready` 固定为 `false`。 + +26 个定向测试覆盖 pending/complete profile、新鲜且完全绑定的合成 attestation、两类 provider integration mode、Gravitino native OIDC/Basic/simple 拒绝、authorization/local evidence/binding drift、placeholder/HTTP/loopback/mutable OCI、敏感字段、自报 claim、过期或失败证明、报告篡改和生产 overclaim。 + +## Claim Boundary + +允许声明： + +- M3-7 production Metadata Fabric identity readiness contract 已建立； +- 当前 pending profile 结构有效，并机器可读地暴露 40 个 blockers； +- 合成完整 profile/attestation 验证了 fail-closed 门禁逻辑。 + +当前不得声明： + +- 已选择、部署或验证生产 IdP、OIDC federation、identity proxy 或 Gravitino custom authenticator； +- 已验证生产 workload identity、TLS/mTLS、tenant isolation、持久 catalog identity binding 或双 provider minimum privilege； +- `production_identity_gate_passed=true` 或 `production_ready=true`。 + +## Consequences + +**Positive**：本地 Basic/JWT evidence 不再可能被误读为生产身份；Gravitino 的实际扩展边界被显式记录；受保护环境可用同一 profile-bound gate 发现 credential、权限、配置和证据漂移。 + +**Negative**：M3-7 本身不增加登录或授权能力。门禁会保持 blocked，直到 Metadata Platform、Security 和 SRE 完成 IdP、provider integration、持久 catalog、tenant、TLS 与运营决策。 + +**Next gate**：批准并物化 production profile，在受保护环境部署 digest-pinned authentication path，生成绑定当前 source/profile 的真实 attestation并通过全部 18 项检查；随后将该身份路径用于受控双 provider ingestion、持久 binding 和 production OpenLineage conformance，且继续独立完成其余 production gates。 diff --git a/docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json b/docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json new file mode 100644 index 00000000..d65acd41 --- /dev/null +++ b/docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json @@ -0,0 +1,152 @@ +{ + "errors": [], + "evidence_fingerprint": "f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd", + "gravitino_authentication_verified": false, + "local_gravitino_basic_identity_verified": true, + "local_gravitino_login_rotation_verified": true, + "local_gravitino_minimum_privilege_verified": true, + "local_gravitino_revocation_verified": true, + "observation": { + "contract": { + "contract_fingerprint": "c1c8b08e52c88742fd273378735491626bc4a185f3416746eafa4932374a3704", + "local_static_contract_verified": true + }, + "gravitino": { + "allowed_probe": { + "create_status": 200, + "full_name": "lakehouse.published.gda_identity_probe", + "operation": "CREATE_TABLE", + "read_status": 200, + "readback_table": "gda_identity_probe", + "resource": "SCHEMA", + "rotated_readback_table": "gda_identity_probe", + "table": "gda_identity_probe" + }, + "authentication": { + "bounded_user_status": 200, + "material_recorded": false, + "service_admin_status": 200, + "unregistered_principal_status": 401 + }, + "bootstrap": { + "catalog": "lakehouse", + "metalake": "gda_identity", + "minimum_privilege": false, + "schema": "published", + "service_admin_used": true + }, + "configuration": { + "access_control_enabled": true, + "authenticator": "basic", + "built_in_idp_scope": "local_poc_only", + "idp_extension": "org.apache.gravitino.idp.web.rest.feature", + "simple_authenticator_trusted": false + }, + "denied_probe": { + "catalog": "unauthorized_catalog", + "operation": "CREATE_CATALOG", + "resource": "METALAKE", + "status": 403 + }, + "idp_create_response_present": true, + "login_lifecycle": { + "after_revocation_status": 401, + "idp_principal_absent": true, + "material_recorded": false, + "old_after_rotation_status": 401, + "revocation_mode": "idp_user_delete", + "rotated_authentication_status": 200, + "rotation_mode": "administrator_reset" + }, + "principal": { + "granted_name": "gda-metadata-projection", + "is_service_admin": false, + "name": "gda-metadata-projection", + "roles": [ + "gda-table-projection" + ] + }, + "role": { + "name": "gda-table-projection", + "readback_name": "gda-table-projection", + "securable_objects": [ + { + "fullName": "lakehouse", + "privileges": [ + { + "condition": "ALLOW", + "name": "USE_CATALOG" + } + ], + "type": "CATALOG" + }, + { + "fullName": "lakehouse.published", + "privileges": [ + { + "condition": "ALLOW", + "name": "CREATE_TABLE" + }, + { + "condition": "ALLOW", + "name": "USE_SCHEMA" + } + ], + "type": "SCHEMA" + } + ] + }, + "version": "1.3.0" + }, + "observed_at": "2026-07-28T15:01:18.018090+00:00", + "runtime": { + "context": "docker-desktop", + "gravitino": { + "image": "gda/gravitino:1.3.0-local-arm64", + "kind": "StatefulSet", + "name": "gravitino-identity", + "ready_replicas": 1, + "service_account": "gravitino-identity", + "service_account_automount_disabled": true, + "uid": "7973b9fc-0383-4749-aad2-7d350b9a73a2" + }, + "namespace": { + "name": "gda-metadata-identity", + "uid": "319ca844-b66a-4d70-b256-4df9134601c7" + }, + "postgresql": { + "ephemeral_data_volume": true, + "image": "postgres:16.10-bookworm", + "kind": "StatefulSet", + "name": "gravitino-identity-postgresql", + "ready_replicas": 1, + "service_account": "gravitino-identity-postgresql", + "service_account_automount_disabled": true, + "uid": "ca890e8d-6bfd-4b82-ba64-c8bce0c69831" + }, + "service": { + "name": "gravitino-identity", + "type": "ClusterIP", + "uid": "a80396a3-a9a0-407c-90c1-a3402278a9a6" + }, + "source_schema_sha256": "7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df" + }, + "runtime_checks": { + "all_port_forwards_stopped": true, + "kubernetes_service_account_used_for_provider_login": false, + "material_recorded": false, + "namespace_absent": true, + "namespace_delete_completed": true, + "provider_objects_retained": false + }, + "schema": "gda.metadata_fabric_gravitino_identity_observation.v1" + }, + "oidc_verified": false, + "production_identity_verified": false, + "production_ready": false, + "protected_workload_identity_verified": false, + "provider_minimum_privilege_verified": false, + "schema": "gda.metadata_fabric_gravitino_identity_evidence.v1", + "status": "local_gravitino_basic_identity_verified", + "tls_verified": false +} diff --git a/docs/roadmap-ar0-platform-truth-2026-07-24.md b/docs/roadmap-ar0-platform-truth-2026-07-24.md index 19cd95f8..b56412b4 100644 --- a/docs/roadmap-ar0-platform-truth-2026-07-24.md +++ b/docs/roadmap-ar0-platform-truth-2026-07-24.md @@ -199,7 +199,7 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 当前完成包括本地合同、授权 evidence、outbox/callback 代码、数据库成功终局门、托管 worker 代码、默认关闭的部署模板、candidate/registry/provenance/release/live observation evidence gate、真实 GHCR subject 与 attestation、受保护 application staging apply/observe、worker readiness blocked evidence、独立单副本 activation admission/workflow、合成 golden slice、真实 PostgreSQL 16 事务边界和 canonical mainline 治理。真实 IAM/OIDC 与 service token 生命周期、worker/callback 单副本运行、live golden slice、独立 DolphinScheduler metadata PostgreSQL、真实 Artifact/Quality/Lineage 终局证据及 production promotion 仍未完成；application 的 `staging_deployed=true` 不代表 worker 已部署，也不改变这些退出门。 -### 4.8 Metadata Fabric Bridge M1 + M2 + M3-5（本地 OpenMetadata 受限身份已验证） +### 4.8 Metadata Fabric Bridge M1 + M2 + M3-7（生产双 provider 身份验收合同已冻结） 第八块回到 AR-1 的 metadata control plane，以 [ADR-036](architecture-decisions/adr-036-read-only-metadata-fabric-bridge-contract.md) 固定 OpenMetadata + Gravitino + GDA Control Ledger 的首条 table slice： @@ -225,8 +225,10 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 20. [ADR-049](architecture-decisions/adr-049-tenant-scoped-metadata-fabric-binding-ledger.md) 已新增 migration 097 与 `PlatformGateway` binding commit/read：真实 provider refs、target/source/definition version、execution-plan、精确 PolicyDecision、独立 Approval 和 provider evidence 必须在同一 tenant 下完整匹配后才可追加。空临时 PostgreSQL 首次提交为 `created=true`、第二次精确 replay 为 `created=false`，FORCE RLS、跨租户不可见和 gateway 无 UPDATE/DELETE 均通过；binding UUID 为 `9580cd65-9fd9-5216-90a5-1fd6837e6cfb`，record SHA 为 `19bdbddedc27d2ed8a35119e8f065a47a02345f9bbd3a51075856cb9587f4176`，evidence SHA 为 `518bfed363aba34e539ada19ea1dc708bacc9eba6578ccab165d11bccfc05223`。M3-3 不调用 provider、不写 legacy，也不覆盖含 synthetic UUID 的既有 Resource。 21. [ADR-050](architecture-decisions/adr-050-idempotent-openlineage-http-delivery.md) 已新增 migration 098 与 tenant-scoped lineage outbox；Gateway 只有在 M3-3 binding、execution-plan 和完整 M3-1 source plan 精确匹配后才可 enqueue。真实 loopback HTTP 演练让接收端先提交事件再返回 503，第二次以同一 `Idempotency-Key` 重发并返回 duplicate 200；共 2 个 wire requests、1 次唯一接受、最终 2 attempts/delivered，完成项不再 claim。delivery UUID 为 `49a54408-b3a8-5843-a27d-6395c080af99`，event SHA 为 `4929e51c4126e09415a9fc1578c9401077c5d7c374294e70deeebd29c8216dd2`，evidence SHA 为 `5aabe9950d2d3d0cfb50ee9d9163c8dee60b04e800f68baad5090e97319b5d8d`。该结论是本地 `at_least_once_with_receiver_idempotency`，`live_openlineage_emission_verified=false`，不是网络 exactly-once 或受保护 production receiver 证明。 22. [ADR-051](architecture-decisions/adr-051-local-openmetadata-bounded-provider-identity.md) 已在 OpenMetadata `1.13.1` 创建临时非管理员 bot，effective roles 精确为 provider 强制 `DefaultBotRole` 加项目专用 role；项目新增 policy 仅允许 `table/Create`。bot 创建/read-back 临时 table 分别返回 201/200，创建 policy 返回 403；JWT 轮换后旧值返回 401，新值返回 200，吊销后新值返回 401。table、bot、user、role、policy 与 denial probe 最终均不存在，evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。该 scoped local 结论不包含 Gravitino、OIDC、Kubernetes-to-provider identity exchange 或生产 credential delivery。 +23. [ADR-052](architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md) 已在隔离 Gravitino `1.3.0` namespace 启用 Basic IdP 与 authorization，bounded user 只获得 `lakehouse` 的 `USE_CATALOG` 以及 `lakehouse.published` 的 `USE_SCHEMA`/`CREATE_TABLE`；table create/read 为 200/200，越权 catalog create 为 403，密码轮换使旧值返回 401，IdP 用户删除使替换值返回 401。临时 metalake/catalog/schema/table/user/role、namespace 与 loopback port-forward 均已清理，证据 SHA 为 `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd`。这只是 Gravitino Basic local POC；probe catalog 使用 memory backend，不包含 OIDC、TLS、Kubernetes-to-provider identity exchange、持久生产 catalog 或双 provider production identity。 +24. [ADR-053](architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md) 已将 OIDC federation、双 provider integration、digest-pinned authentication component、workload/tenant claim、Kubernetes ServiceAccount、禁止 direct bypass、short-lived token、M3-5/M3-6 精确 allow/deny contract、TLS/mTLS、持久 Gravitino catalog、tenant isolation、owner/audit/SLO/runbook 与 18 项 protected attestation check 冻结为 fail-closed profile。Gravitino `1.3.0` 镜像只发现 Basic IdP jar，因此生产只允许明确选择并证明 `custom_oidc_authenticator` 或 `identity_aware_proxy`，不假设 native OIDC。当前 profile fingerprint 为 `2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`，合同有效但 40 项外部输入 blocked，没有真实 attestation，全部 production identity claims 仍为 `false`。 -此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上新增的 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝。M3-2 ingestion 仍使用 bootstrap admin，Gravitino 未认证且 catalog backend 为 memory，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、持久 catalog、TLS、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、两项 production gate 和 `production_ready` 仍为 `false`。 +此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上的项目新增 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝；M3-6 只证明隔离 Gravitino Basic IdP 的 bounded table-create、catalog-create 拒绝、登录轮换/吊销和完整清理；M3-7 只证明 pending production identity profile、profile-bound attestation 和派生 claim 的 fail-closed 合同可校验，没有部署或证明真实身份路径。M3-2 ingestion 仍使用 bootstrap admin，Gravitino M3-6 probe catalog 使用 memory backend，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、TLS、持久 catalog、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、三项 production gate 和 `production_ready` 仍为 `false`。 ## 5. 重新评估条件 diff --git a/docs/roadmap.md b/docs/roadmap.md index aefad007..df27586b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -417,7 +417,7 @@ AR-0 Architecture / Schema / Runtime Truth - Web、DolphinScheduler、OpenMetadata、Gravitino、worker 和 bridge 重启后不丢版本、产品、血缘、PlatformRun correlation 或审批状态。 - DataOps 状态可从 DolphinScheduler/OpenMetadata/Gravitino/GDA evidence 恢复；任何 release/deployment 都能从事件、Artifact、评测、策略、incident 和 rollback pointer 重放。Temporal durable recovery 在 AR-5/AR-7 单独验收。 -当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 又以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401。这个 `local_openmetadata_minimum_privilege_verified=true` 明确限定在 provider 强制 `DefaultBotRole` 之上的项目新增 grant；M3-2 ingestion 仍使用 bootstrap admin，Gravitino 仍未认证且使用 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、持久 catalog、生产 ingestion/conformance、生产 lineage receiver 与 `production_ready` 仍为 false。 +当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401；ADR-052 又在隔离 Gravitino `1.3.0` Basic IdP 中证明 bounded user 的 `USE_CATALOG`、`USE_SCHEMA`、`CREATE_TABLE` 范围、catalog-create 403、密码轮换和用户吊销；ADR-053 将生产 OIDC federation、双 provider integration/workload identity、最小权限、TLS/mTLS、持久 Gravitino catalog、tenant isolation、运营责任和新鲜 protected attestation 冻结为 fail-closed readiness contract。Gravitino `1.3.0` 镜像只发现 Basic IdP，不假设 native OIDC；当前 profile 仍有 40 个外部 blockers 且未提交真实 attestation。`local_openmetadata_minimum_privilege_verified=true` 与 `local_gravitino_minimum_privilege_verified=true` 都只描述各自临时 provider rehearsal；M3-2 ingestion 仍使用 bootstrap admin，Gravitino probe catalog 仍是 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、TLS、持久 catalog、生产 ingestion/conformance、生产 lineage receiver、`production_identity_gate_passed` 与 `production_ready` 仍为 false。 ### AR-2 — Source, Ingestion and Geospatial Lakehouse Vertical Slice（P0） diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index a2cd9cb4..199c6ef7 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -2,7 +2,7 @@ 日期：2026-07-28 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery 与 M3-5 local OpenMetadata bounded identity 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery、M3-5 local OpenMetadata bounded identity、M3-6 local Gravitino Basic bounded identity 与 M3-7 production identity readiness contract 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation、生产 identity attestation 和生产切换仍 `in_progress` 适用分支：`main` @@ -34,7 +34,7 @@ | 事件交付 | Standards outbox 已数据库耐久；`gda_control.platform_command_outbox` 已为 DolphinScheduler dispatch/reconcile 提供 tenant RLS、lease claim、幂等 callback、薄 consumer library、managed worker process 和受保护的单副本 staging activation boundary；其他 WebSocket/bot/feedback 多为 best effort | command delivery status、消费者 claim、worker status JSON、activation/readiness evidence、WebSocket 消息 | command/event 与源事实同事务入 outbox，幂等 consumer 交付；worker status、outbox 状态、activation evidence、缓存或 socket 都不是 Run/业务权威 | Platform/Integrations | AR-1 command delivery/activation 代码已验证 -> worker/callback live evidence 待完成 | | 质量结果 | `gda_control.quality_result` 已提供 tenant RLS、append-only gateway 写入，绑定 Run、output ResourceVersion、rule version、verdict、metrics、evidence Artifact 和独立 evaluator；standards、QC、MMFE 专项结果仍未迁移 | dashboard、OpenMetadata quality summary | GDA ledger 保存产品终局所需的不可变 verdict/evidence；OpenMetadata 与 UI 只作可重建发现投影；旧结果缺稳定版本和证据时不得升级为终局依据 | Governance/DataOps | AR-1 最小成功证据已验证 -> 真实规则/staging 待接入 | | 标准与语义定义 | `std_*`、semantic registry 和 YAML 共同存在，生命周期未统一 | prompt/context、搜索索引 | 版本化 Standard/SemanticDefinition 经审批后为权威；Agent context 只消费批准版本 | Governance | AR-1 -> AR-3 | -| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5 本地 OpenMetadata bot 已验证 scoped project grant、越权拒绝和 JWT 轮换/吊销，但由 bootstrap admin 临时 provision，Gravitino 未认证 | session/cache、前端菜单权限、本地 provider identity evidence | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；完成受保护 OIDC/IAM provisioning、双 provider 最小权限、secret delivery 和轮换/吊销后才能生产切换 | Security | AR-1 local OpenMetadata identity 已验证 -> protected 双 provider IAM 待验收 | +| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5/M3-6 分别验证本地 provider scoped grant、越权拒绝和 credential rotation/revocation；M3-7 已冻结生产 OIDC/workload/tenant binding、TLS、持久 catalog 与 attestation contract，但 40 个外部输入仍 blocked | session/cache、前端菜单权限、本地 provider identity evidence、pending profile 与合成 readiness report | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；只有 fresh protected attestation 可派生双 provider production identity claims，profile、Basic/JWT evidence 或人工批准均不可替代 | Security | AR-1 local identities + production readiness contract 已验证 -> protected 双 provider IAM/attestation 待执行 | | GIS 服务定义与 active revision | Martin、REST/MVT/STAC endpoints 和配置直接暴露 | Ingress、tile cache、客户端图层 | GIS Service Control Plane 管 Service/Layer/Style/TMS/DeploymentRevision；provider/Gateway 仅执行 | GIS Platform | AR-4 | | 缓存与进度 | Redis、进程内 dict/task map | UI progress、tile/context cache | 永不作为资产、Run、workflow 或产品权威；丢失后必须从 ledger/provider 重建 | SRE/Platform | 持续约束 | | Agent/Prompt/Model bundle | 多个 registry 与 YAML 存在，尚无统一 deployment revision | trace、eval dashboard | AgentSpecBundle + EvaluationBinding + DeploymentRevision；只消费已发布 DataProductVersion | AgentOps | AR-5 | @@ -91,6 +91,8 @@ - Metadata Fabric M3-3 已新增 tenant-scoped append-only binding ledger，将 M3-2 的真实 OpenMetadata UUID、Gravitino ref、execution-plan、PolicyDecision、Approval 与 provider evidence 经 PlatformGateway 精确校验后落账；空临时 PostgreSQL 首次提交 `created=true`、replay `created=false`，跨租户读取、UPDATE/DELETE 均拒绝。binding SHA 为 `125d7197f05ff9c37999a94d090d123dcf905480b776da0738d9625ab5045598`，record SHA 为 `19bdbddedc27d2ed8a35119e8f065a47a02345f9bbd3a51075856cb9587f4176`，evidence SHA 为 `518bfed363aba34e539ada19ea1dc708bacc9eba6578ccab165d11bccfc05223`。该演练不调用 provider，`provider_minimum_privilege_verified=false`、`oidc_verified=false`、`durable_catalog_verified=false`、`live_openlineage_emission_verified=false`、`production_ingestion_verified=false`、`production_ready=false`。 - Metadata Fabric M3-4 已新增 tenant-scoped lineage outbox、Gateway claim/complete/fail 和严格 loopback HTTP emitter；接收端第一次提交 event 后返回 503，第二次以同一 idempotency key/body 重发并只返回 duplicate 200，形成 2 wire requests、1 unique accept、2 attempts 和不可再 claim 的 delivered receipt。delivery UUID 为 `49a54408-b3a8-5843-a27d-6395c080af99`，event SHA 为 `4929e51c4126e09415a9fc1578c9401077c5d7c374294e70deeebd29c8216dd2`，evidence SHA 为 `5aabe9950d2d3d0cfb50ee9d9163c8dee60b04e800f68baad5090e97319b5d8d`。`local_wire_openlineage_delivery_verified=true` 仅表示真实本机 HTTP wire；`live_openlineage_emission_verified=false`、`provider_minimum_privilege_verified=false`、`oidc_verified=false`、`durable_catalog_verified=false`、`production_receiver_verified=false`、`production_ingestion_verified=false`、`production_ready=false`。 - Metadata Fabric M3-5 已创建临时 OpenMetadata 非管理员 bot，effective roles 只有 provider 强制 `DefaultBotRole` 与项目 role，项目 policy 只有 `table/Create`；table create/read 为 201/200，policy create 为 403，旧 JWT 轮换后为 401，新 JWT 吊销后为 401，六项 provider cleanup 与 port-forward cleanup 全部通过。evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。`local_openmetadata_minimum_privilege_verified=true` 只描述强制 default role 之上的项目新增 grant；`provider_minimum_privilege_verified=false`、`protected_workload_identity_verified=false`、`oidc_verified=false`、`gravitino_authentication_verified=false`、`production_identity_verified=false`、`production_ready=false`。 +- Metadata Fabric M3-6 已在隔离 Gravitino `1.3.0` Basic IdP 中将 bounded user 限定为 `lakehouse` 的 `USE_CATALOG` 与 `lakehouse.published` 的 `USE_SCHEMA`/`CREATE_TABLE`；table create/read 为 200/200，catalog create 为 403，旧密码轮换后为 401，用户删除后替换密码为 401，临时 provider 对象、namespace 和 port-forward 全部清理。evidence SHA 为 `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd`。`local_gravitino_minimum_privilege_verified=true` 只描述本地 Basic rehearsal；OIDC、TLS、持久 catalog 和 production identity 均仍为 `false`。 +- Metadata Fabric M3-7 已建立 production identity profile/attestation gate；checked-in profile fingerprint 为 `2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`，report fingerprint 为 `c607589ee25a87acc8a1ab71372618a9a4c10c1e8ebff15b8db7e78b37600b9f`，`profile_valid=true`，40 项 federation/provider/TLS/catalog/tenancy/operations 外部输入以 blockers 暴露，`ready_for_protected_verification=false`、`production_identity_gate_passed=false`、`production_ready=false`。合成完整 attestation 只验证门禁逻辑，不计入生产证据。 ## 下一验收证据 @@ -98,5 +100,5 @@ - staging 的 migration role、应用 login membership、连接池 role/tenant 复位、双租户 API 和 success finalization 运行产物； - 为受保护 activation 提供真实 ConfigMap snapshot、Secret key attestation、provider identity 和 reviewer approval，随后完成 managed outbox worker/provider callback 单副本实际部署、唯一 worker ID、status/lease 故障恢复和无双写证据； - 首条真实图斑链对 golden slice 的 output hash、独立质量结果/evidence、血缘、发布 revision 和 rollback 演练； -- OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery 与 M3-5 本地临时 OpenMetadata identity 均不计入生产退出门； +- OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery、M3-5/M3-6 本地临时 provider identity 与 M3-7 pending profile/合成 attestation 均不计入生产退出门； - DolphinScheduler/Temporal sandbox 的独立数据库、备份恢复、身份、版本和升级责任证明；DolphinScheduler standalone/H2 不计入此退出门。 diff --git a/k8s/metadata-fabric-gravitino-identity/kustomization.yaml b/k8s/metadata-fabric-gravitino-identity/kustomization.yaml new file mode 100644 index 00000000..fc965874 --- /dev/null +++ b/k8s/metadata-fabric-gravitino-identity/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: gda-metadata-identity +resources: + - namespace.yaml + - runtime.yaml +labels: + - pairs: + app.kubernetes.io/part-of: gda-metadata-fabric-gravitino-identity + gda.openai.com/environment: identity-rehearsal + includeSelectors: false diff --git a/k8s/metadata-fabric-gravitino-identity/namespace.yaml b/k8s/metadata-fabric-gravitino-identity/namespace.yaml new file mode 100644 index 00000000..245f0b39 --- /dev/null +++ b/k8s/metadata-fabric-gravitino-identity/namespace.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gda-metadata-identity + labels: + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: gravitino-identity-rehearsal + namespace: gda-metadata-identity +spec: + hard: + requests.cpu: "2" + requests.memory: 3Gi + limits.cpu: "4" + limits.memory: 6Gi + pods: "4" + secrets: "4" + configmaps: "4" diff --git a/k8s/metadata-fabric-gravitino-identity/runtime.yaml b/k8s/metadata-fabric-gravitino-identity/runtime.yaml new file mode 100644 index 00000000..00dbe47d --- /dev/null +++ b/k8s/metadata-fabric-gravitino-identity/runtime.yaml @@ -0,0 +1,259 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gravitino-identity-postgresql + namespace: gda-metadata-identity +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gravitino-identity + namespace: gda-metadata-identity +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gravitino-identity-config + namespace: gda-metadata-identity +data: + gravitino.conf: | + gravitino.server.shutdown.timeout = 30000 + gravitino.server.webserver.host = 0.0.0.0 + gravitino.server.webserver.httpPort = 8090 + gravitino.server.health.entityStore.probeTimeoutMs = 5000 + gravitino.entity.store = relational + gravitino.entity.store.relational = JDBCBackend + gravitino.entity.store.relational.jdbcUrl = jdbc:postgresql://gravitino-identity-postgresql:5432/gravitino + gravitino.entity.store.relational.jdbcDriver = org.postgresql.Driver + gravitino.entity.store.relational.jdbcUser = gravitino + gravitino.authenticators = basic + gravitino.server.rest.extensionPackages = org.apache.gravitino.idp.web.rest.feature + gravitino.authorization.enable = true + gravitino.authorization.serviceAdmins = gda-identity-admin + gravitino.cache.enabled = false + gravitino.auxService.names = +--- +apiVersion: v1 +kind: Service +metadata: + name: gravitino-identity-postgresql + namespace: gda-metadata-identity +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: gravitino-identity-postgresql + ports: + - name: postgresql + port: 5432 + targetPort: postgresql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: gravitino-identity-postgresql + namespace: gda-metadata-identity +spec: + serviceName: gravitino-identity-postgresql + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: gravitino-identity-postgresql + template: + metadata: + labels: + app.kubernetes.io/name: gravitino-identity-postgresql + app.kubernetes.io/component: database + spec: + serviceAccountName: gravitino-identity-postgresql + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + securityContext: + fsGroup: 999 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: postgresql + image: postgres:16.10-bookworm + imagePullPolicy: IfNotPresent + env: + - name: POSTGRES_USER + value: gravitino + - name: POSTGRES_DB + value: gravitino + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: gravitino-identity-runtime + key: database-password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgresql + containerPort: 5432 + startupProbe: + exec: + command: ["pg_isready", "-U", "gravitino", "-d", "gravitino"] + periodSeconds: 3 + failureThreshold: 60 + readinessProbe: + exec: + command: ["pg_isready", "-U", "gravitino", "-d", "gravitino"] + periodSeconds: 5 + failureThreshold: 12 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: runtime + mountPath: /var/run/postgresql + - name: tmp + mountPath: /tmp + - name: schema + mountPath: /docker-entrypoint-initdb.d/001-schema.sql + subPath: 001-schema.sql + readOnly: true + volumes: + - name: data + emptyDir: {} + - name: runtime + emptyDir: {} + - name: tmp + emptyDir: {} + - name: schema + configMap: + name: gravitino-identity-schema +--- +apiVersion: v1 +kind: Service +metadata: + name: gravitino-identity + namespace: gda-metadata-identity +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: gravitino-identity + ports: + - name: http + port: 8090 + targetPort: http +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: gravitino-identity + namespace: gda-metadata-identity +spec: + serviceName: gravitino-identity + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: gravitino-identity + template: + metadata: + labels: + app.kubernetes.io/name: gravitino-identity + app.kubernetes.io/component: technical-metadata + spec: + serviceAccountName: gravitino-identity + automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: gravitino + image: gda/gravitino:1.3.0-local-arm64 + imagePullPolicy: Never + command: ["/bin/bash", "-ec"] + args: + - | + install -d -m 0700 /var/run/gravitino/conf + install -m 0600 /etc/gravitino/gravitino.conf /var/run/gravitino/conf/gravitino.conf + install -m 0444 /opt/gravitino/conf/gravitino-env.sh /var/run/gravitino/conf/gravitino-env.sh + printf '\ngravitino.entity.store.relational.jdbcPassword = ' >> /var/run/gravitino/conf/gravitino.conf + cat /var/run/secrets/gravitino/database-password >> /var/run/gravitino/conf/gravitino.conf + printf '\n' >> /var/run/gravitino/conf/gravitino.conf + exec /opt/gravitino/bin/gravitino.sh --config /var/run/gravitino/conf run + env: + - name: GRAVITINO_MEM + value: -Xms384m -Xmx1024m -XX:MaxMetaspaceSize=384m + - name: GRAVITINO_INITIAL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: gravitino-identity-runtime + key: admin-password + ports: + - name: http + containerPort: 8090 + startupProbe: + httpGet: + path: /api/health/ready + port: http + periodSeconds: 5 + failureThreshold: 120 + readinessProbe: + httpGet: + path: /api/health/ready + port: http + periodSeconds: 10 + failureThreshold: 12 + resources: + requests: + cpu: 250m + memory: 768Mi + limits: + cpu: "2" + memory: 2Gi + securityContext: + runAsNonRoot: true + runAsUser: 1000 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: config + mountPath: /etc/gravitino + readOnly: true + - name: secrets + mountPath: /var/run/secrets/gravitino + readOnly: true + - name: runtime + mountPath: /var/run/gravitino + - name: logs + mountPath: /opt/gravitino/logs + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: gravitino-identity-config + - name: secrets + secret: + secretName: gravitino-identity-runtime + defaultMode: 0400 + - name: runtime + emptyDir: {} + - name: logs + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/scripts/metadata-fabric-gravitino-identity.sh b/scripts/metadata-fabric-gravitino-identity.sh new file mode 100755 index 00000000..fdd9f92a --- /dev/null +++ b/scripts/metadata-fabric-gravitino-identity.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMMON_GIT_DIR="$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +SHARED_ROOT="" +if [ -n "$COMMON_GIT_DIR" ]; then + SHARED_ROOT="$(cd "$COMMON_GIT_DIR/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$ROOT/.venv/bin/python" ]; then + PYTHON="$ROOT/.venv/bin/python" +elif [ -n "$SHARED_ROOT" ] && [ -x "$SHARED_ROOT/.venv/bin/python" ]; then + PYTHON="$SHARED_ROOT/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$ROOT" +exec "$PYTHON" -m data_agent.metadata_fabric_gravitino_identity "$@" diff --git a/scripts/metadata-fabric-identity-gate.sh b/scripts/metadata-fabric-identity-gate.sh new file mode 100755 index 00000000..81482009 --- /dev/null +++ b/scripts/metadata-fabric-identity-gate.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMMON_GIT_DIR="$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +SHARED_ROOT="" +if [ -n "$COMMON_GIT_DIR" ]; then + SHARED_ROOT="$(cd "$COMMON_GIT_DIR/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$ROOT/.venv/bin/python" ]; then + PYTHON="$ROOT/.venv/bin/python" +elif [ -n "$SHARED_ROOT" ] && [ -x "$SHARED_ROOT/.venv/bin/python" ]; then + PYTHON="$SHARED_ROOT/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$ROOT" +exec "$PYTHON" -m data_agent.metadata_fabric_identity_gate "$@"