diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index de2d7c38..55feb84f 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -169,6 +169,7 @@ jobs: data_agent/test_chongqing_extraction_provenance.py \ data_agent/test_chongqing_source_governance.py \ data_agent/test_chongqing_admission_readiness.py \ + data_agent/test_chongqing_protected_admission.py \ data_agent/test_dolphinscheduler_adapter.py \ data_agent/test_dolphinscheduler_command_consumer.py \ data_agent/test_dolphinscheduler_command_worker.py \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8147f03e..1e347c65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,6 +136,9 @@ jobs: - name: Validate Chongqing admission readiness evidence run: python -m data_agent.chongqing_admission_readiness + - name: Validate Chongqing protected admission intake + run: python -m data_agent.chongqing_protected_admission validate + - name: Validate DolphinScheduler adapter boundary run: python -m data_agent.dolphinscheduler_adapter validate @@ -175,6 +178,7 @@ jobs: data_agent/test_chongqing_extraction_provenance.py \ data_agent/test_chongqing_source_governance.py \ data_agent/test_chongqing_admission_readiness.py \ + data_agent/test_chongqing_protected_admission.py \ data_agent/test_dolphinscheduler_adapter.py \ data_agent/test_dolphinscheduler_command_consumer.py \ data_agent/test_dolphinscheduler_command_worker.py \ diff --git a/data_agent/chongqing_protected_admission.py b/data_agent/chongqing_protected_admission.py new file mode 100644 index 00000000..20f055fc --- /dev/null +++ b/data_agent/chongqing_protected_admission.py @@ -0,0 +1,384 @@ +"""Evaluate the protected attestation intake for Chongqing admission. + +M3-32 consumes the metadata-only M3-31 readiness record and an external, +protected attestation bundle. It verifies binding, freshness and the complete +fifteen-item evidence inventory without reading or copying source payloads. +The evaluator only produces a report; it never creates Landing, ResourceVersion, +PlatformRun, scheduler or provider authority. +""" + +from __future__ import annotations + +import argparse +import hashlib +import ipaddress +import json +import re +import sys +from collections.abc import Mapping +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from . import chongqing_admission_readiness as readiness +from . import chongqing_real_source_admission as admission + +ATTESTATION_SCHEMA = "gda.chongqing_protected_admission_attestation.v1" +REPORT_SCHEMA = "gda.chongqing_protected_admission_report.v1" +PROTECTED_ENVIRONMENT = "chongqing-admission-protected" +MAX_ATTESTATION_AGE = timedelta(hours=24) +MAX_ATTESTATION_LIFETIME = timedelta(days=7) + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_READINESS_PATH = readiness.DEFAULT_EVIDENCE_PATH + +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +PLACEHOLDER_PATTERN = re.compile( + r"(^|[-_.:/])(pending|placeholder|replace|tbd|todo|changeme)([-_.:/]|$)|[<>]", + re.IGNORECASE, +) + +EXPECTED_CHECKS = { + "archive_hash_match", + "extracted_hash_match", + "derivation_evidence_complete", + "governance_decisions_complete", + "source_binding_match", + "payload_and_path_free", + "reviewer_approval", + "provider_mutation_forbidden", +} +REQUIRED_ATTESTATION_KEYS = { + "schema", + "readiness_evidence_sha256", + "readiness_evidence_file_sha256", + "source_binding", + "observed_at", + "expires_at", + "protected_environment", + "verifier_identity", + "evidence_uri", + "requirements", + "checks", + "attestation_sha256", +} +REQUIRED_SOURCE_BINDING_KEYS = {"source_id", "source_group_id", "asset_id", "source_ref"} +REQUIRED_REQUIREMENT_RECORD_KEYS = {"status", "attestation_sha256"} + + +class ChongqingProtectedAdmissionError(RuntimeError): + """The protected attestation intake failed closed.""" + + +def canonical_json_fingerprint(value: Any) -> str: + payload = json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _load_json_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("JSON document is not an object") + return value + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _placeholder(value: Any) -> bool: + return not isinstance(value, str) or not value.strip() or bool( + PLACEHOLDER_PATTERN.search(value.strip()) + ) + + +def _safe_https_uri(value: Any) -> bool: + if _placeholder(value): + return False + parsed = urlparse(str(value)) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + ): + return False + hostname = parsed.hostname.lower() + if hostname in {"localhost", "0.0.0.0", "::1"} or hostname.endswith( + (".localhost", ".local", ".svc", ".cluster.local", ".example", ".invalid", ".test") + ): + return False + if hostname in {"example.com", "example.net", "example.org"}: + return False + try: + if ipaddress.ip_address(hostname).is_loopback: + return False + except ValueError: + pass + return True + + +def _expected_source_binding(readiness_evidence: Mapping[str, Any]) -> dict[str, Any]: + source = _mapping(readiness_evidence.get("source_binding")) + return { + "source_id": source.get("source_id"), + "source_group_id": source.get("source_group_id"), + "asset_id": source.get("asset_id"), + "source_ref": source.get("source_ref"), + } + + +def _readiness_errors( + readiness_evidence: Mapping[str, Any], + readiness_file_sha256: str | None, +) -> list[str]: + errors = readiness.validate_evidence(readiness_evidence) + if readiness_file_sha256 != readiness.EVIDENCE_FILE_SHA256: + errors.append("M3-31 readiness evidence file fingerprint does not match") + return sorted(set(errors)) + + +def _attestation_errors( + attestation: Mapping[str, Any] | None, + *, + readiness_evidence: Mapping[str, Any], + readiness_evidence_sha256: str, + readiness_file_sha256: str | None, + now: datetime, + max_age: timedelta, +) -> list[str]: + if attestation is None: + return ["protected admission attestation is missing"] + + errors: list[str] = [] + if readiness._path_or_payload_findings(attestation): + errors.append("protected admission attestation contains a path or payload marker") + if set(attestation) != REQUIRED_ATTESTATION_KEYS: + errors.append("protected admission attestation inventory does not match") + stable = {key: value for key, value in attestation.items() if key != "attestation_sha256"} + if attestation.get("attestation_sha256") != canonical_json_fingerprint(stable): + errors.append("protected admission attestation fingerprint does not match") + if attestation.get("schema") != ATTESTATION_SCHEMA: + errors.append("protected admission attestation schema does not match") + if attestation.get("readiness_evidence_sha256") != readiness_evidence_sha256: + errors.append("protected admission attestation is not bound to M3-31 evidence") + if attestation.get("readiness_evidence_file_sha256") != readiness_file_sha256: + errors.append("protected admission attestation is not bound to M3-31 evidence file") + if not SHA256_PATTERN.fullmatch(str(attestation.get("readiness_evidence_sha256") or "")): + errors.append("protected admission readiness evidence fingerprint is invalid") + if not SHA256_PATTERN.fullmatch(str(attestation.get("readiness_evidence_file_sha256") or "")): + errors.append("protected admission readiness file fingerprint is invalid") + + source_binding = _mapping(attestation.get("source_binding")) + if set(source_binding) != REQUIRED_SOURCE_BINDING_KEYS: + errors.append("protected admission source binding inventory does not match") + if dict(source_binding) != _expected_source_binding(readiness_evidence): + errors.append("protected admission source binding does not match M3-31") + if not admission.SOURCE_REF_PATTERN.fullmatch(str(source_binding.get("source_ref") or "")): + errors.append("protected admission source reference is invalid") + + if attestation.get("protected_environment") != PROTECTED_ENVIRONMENT: + errors.append("protected admission environment does not match") + if _placeholder(attestation.get("verifier_identity")): + errors.append("protected admission verifier identity is missing") + if not _safe_https_uri(attestation.get("evidence_uri")): + errors.append("protected admission evidence URI is invalid") + + try: + observed_at = datetime.fromisoformat(str(attestation.get("observed_at"))) + expires_at = datetime.fromisoformat(str(attestation.get("expires_at"))) + if observed_at.tzinfo is None or observed_at.utcoffset() is None: + raise ValueError + if expires_at.tzinfo is None or expires_at.utcoffset() is None: + raise ValueError + age = now - observed_at + if age < timedelta(seconds=-30) or age > max_age: + errors.append("protected admission attestation is outside the freshness window") + if expires_at <= now or expires_at <= observed_at: + errors.append("protected admission attestation has expired or invalid expiry") + if expires_at - observed_at > MAX_ATTESTATION_LIFETIME: + errors.append("protected admission attestation lifetime exceeds seven days") + except ValueError: + errors.append("protected admission attestation timestamps are invalid") + + requirements = _mapping(attestation.get("requirements")) + if set(requirements) != set(readiness.REQUIREMENT_KEYS): + errors.append("protected admission requirement inventory does not match") + for key in readiness.REQUIREMENT_KEYS: + record = _mapping(requirements.get(key)) + if set(record) != REQUIRED_REQUIREMENT_RECORD_KEYS: + errors.append(f"protected admission requirement record does not match: {key}") + continue + if record.get("status") != "verified": + errors.append(f"protected admission requirement is not verified: {key}") + if not SHA256_PATTERN.fullmatch(str(record.get("attestation_sha256") or "")): + errors.append(f"protected admission requirement attestation is invalid: {key}") + + checks = _mapping(attestation.get("checks")) + if set(checks) != EXPECTED_CHECKS: + errors.append("protected admission check inventory does not match") + for check in sorted(EXPECTED_CHECKS): + if checks.get(check) != "passed": + errors.append(f"protected admission check did not pass: {check}") + return sorted(set(errors)) + + +def build_admission_report( + *, + readiness_path: Path | None = None, + attestation: Mapping[str, Any] | None = None, + now: datetime | None = None, + max_attestation_age: timedelta = MAX_ATTESTATION_AGE, +) -> dict[str, Any]: + """Build a deterministic report without granting any write authority.""" + current = now or datetime.now(UTC) + if current.tzinfo is None or current.utcoffset() is None: + raise ChongqingProtectedAdmissionError("verification time must be timezone-aware") + if max_attestation_age <= timedelta(0): + raise ChongqingProtectedAdmissionError("attestation freshness window must be positive") + + path = (readiness_path or DEFAULT_READINESS_PATH).resolve() + try: + readiness_file_sha256 = _file_sha256(path) + readiness_evidence = _load_json_object(path) + readiness_evidence_sha256 = str(readiness_evidence.get("evidence_sha256") or "") + readiness_errors = _readiness_errors(readiness_evidence, readiness_file_sha256) + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + readiness_file_sha256 = None + readiness_evidence = {} + readiness_evidence_sha256 = "" + readiness_errors = [f"M3-31 readiness evidence is unreadable: {type(exc).__name__}"] + + readiness_valid = not readiness_errors + attestation_errors = _attestation_errors( + attestation, + readiness_evidence=readiness_evidence, + readiness_evidence_sha256=readiness_evidence_sha256, + readiness_file_sha256=readiness_file_sha256, + now=current, + max_age=max_attestation_age, + ) + attestation_valid = readiness_valid and not attestation_errors + admission_eligible = readiness_valid and attestation_valid + stable = { + "schema": REPORT_SCHEMA, + "readiness_evidence_sha256": readiness_evidence_sha256 or None, + "readiness_evidence_file_sha256": readiness_file_sha256, + "readiness_valid": readiness_valid, + "readiness_errors": readiness_errors, + "attestation_fingerprint": ( + canonical_json_fingerprint(attestation) if attestation is not None else None + ), + "attestation_valid": attestation_valid, + "attestation_errors": attestation_errors, + "admission_eligible": admission_eligible, + "content_admission_authorized": False, + "source_content_admitted": False, + "landing_authority_created": False, + "resource_version_created": False, + "platform_run_created": False, + "scheduler_submission_authorized": False, + "provider_mutation_authorized": False, + "production_ready": False, + } + return {**stable, "report_fingerprint": canonical_json_fingerprint(stable)} + + +def verify_report_integrity(report: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if readiness._path_or_payload_findings(report): + errors.append("protected admission report contains a path or payload marker") + if report.get("schema") != REPORT_SCHEMA: + errors.append("protected admission report schema does not match") + stable = {key: value for key, value in report.items() if key != "report_fingerprint"} + if report.get("report_fingerprint") != canonical_json_fingerprint(stable): + errors.append("protected admission report fingerprint does not match") + if report.get("production_ready") is not False: + errors.append("protected admission report may not claim production readiness") + for key in ( + "content_admission_authorized", + "source_content_admitted", + "landing_authority_created", + "resource_version_created", + "platform_run_created", + "scheduler_submission_authorized", + "provider_mutation_authorized", + ): + if report.get(key) is not False: + errors.append(f"protected admission report may not claim authority: {key}") + expected_eligible = report.get("readiness_valid") is True and report.get( + "attestation_valid" + ) is True + if report.get("admission_eligible") is not expected_eligible: + errors.append("protected admission eligibility is inconsistent") + return sorted(set(errors)) + + +def _write_report(report: Mapping[str, Any], output: Path | None) -> None: + rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if output is None: + print(rendered, end="") + else: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate_parser = subparsers.add_parser("validate") + validate_parser.add_argument("--readiness", type=Path, default=DEFAULT_READINESS_PATH) + evaluate_parser = subparsers.add_parser("evaluate") + evaluate_parser.add_argument("--readiness", type=Path, default=DEFAULT_READINESS_PATH) + evaluate_parser.add_argument("--attestation", type=Path, required=True) + evaluate_parser.add_argument("--output", type=Path) + verify_parser = subparsers.add_parser("verify") + verify_parser.add_argument("--input", type=Path, required=True) + args = parser.parse_args(argv) + + try: + if args.command == "validate": + report = build_admission_report(readiness_path=args.readiness) + _write_report(report, None) + return 0 if report["readiness_valid"] else 1 + if args.command == "evaluate": + attestation = _load_json_object(args.attestation) + report = build_admission_report( + readiness_path=args.readiness, + attestation=attestation, + ) + _write_report(report, args.output) + return 0 if report["admission_eligible"] else 1 + report = _load_json_object(args.input) + errors = verify_report_integrity(report) + _write_report({"verified": not errors, "errors": errors}, None) + return 0 if not errors else 1 + except ( + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + ChongqingProtectedAdmissionError, + ) as exc: + print(f"Chongqing protected admission: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/test_chongqing_protected_admission.py b/data_agent/test_chongqing_protected_admission.py new file mode 100644 index 00000000..504cda2d --- /dev/null +++ b/data_agent/test_chongqing_protected_admission.py @@ -0,0 +1,141 @@ +import json +from copy import deepcopy +from datetime import UTC, datetime, timedelta + +from data_agent import chongqing_admission_readiness as readiness +from data_agent import chongqing_protected_admission as protected + +NOW = datetime(2026, 8, 17, 12, 0, tzinfo=UTC) + + +def _readiness() -> dict: + return json.loads(readiness.DEFAULT_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def _attestation() -> dict: + evidence = _readiness() + source = evidence["source_binding"] + value = { + "schema": protected.ATTESTATION_SCHEMA, + "readiness_evidence_sha256": evidence["evidence_sha256"], + "readiness_evidence_file_sha256": readiness.EVIDENCE_FILE_SHA256, + "source_binding": { + key: source[key] + for key in ("source_id", "source_group_id", "asset_id", "source_ref") + }, + "observed_at": "2026-08-17T11:45:00Z", + "expires_at": "2026-08-18T11:45:00Z", + "protected_environment": protected.PROTECTED_ENVIRONMENT, + "verifier_identity": "protected-admission-verifier@platform", + "evidence_uri": "https://attestations.gisplatform.com/admission/2026-08-17", + "requirements": { + key: {"status": "verified", "attestation_sha256": "a" * 64} + for key in readiness.REQUIREMENT_KEYS + }, + "checks": {key: "passed" for key in protected.EXPECTED_CHECKS}, + } + stable = dict(value) + value["attestation_sha256"] = protected.canonical_json_fingerprint(stable) + return value + + +def test_pending_profile_is_valid_but_protected_attestation_is_missing(): + report = protected.build_admission_report(now=NOW) + + assert report["readiness_valid"] is True + assert report["attestation_valid"] is False + assert report["admission_eligible"] is False + assert report["attestation_errors"] == [ + "protected admission attestation is missing" + ] + assert report["content_admission_authorized"] is False + assert report["production_ready"] is False + + +def test_complete_attestation_only_passes_the_readiness_evaluation(): + report = protected.build_admission_report(attestation=_attestation(), now=NOW) + + assert report["readiness_valid"] is True + assert report["attestation_valid"] is True + assert report["admission_eligible"] is True + assert report["content_admission_authorized"] is False + assert report["source_content_admitted"] is False + assert report["landing_authority_created"] is False + assert report["resource_version_created"] is False + assert report["platform_run_created"] is False + assert report["scheduler_submission_authorized"] is False + assert report["provider_mutation_authorized"] is False + assert report["production_ready"] is False + assert protected.verify_report_integrity(report) == [] + + +def test_attestation_binding_drift_fails_closed(): + attestation = _attestation() + attestation["source_binding"]["asset_id"] = "other_asset" + attestation["attestation_sha256"] = protected.canonical_json_fingerprint( + {key: value for key, value in attestation.items() if key != "attestation_sha256"} + ) + + report = protected.build_admission_report(attestation=attestation, now=NOW) + + assert report["admission_eligible"] is False + assert "protected admission source binding does not match M3-31" in report[ + "attestation_errors" + ] + + +def test_stale_attestation_fails_closed(): + attestation = _attestation() + attestation["observed_at"] = "2026-08-15T11:45:00Z" + attestation["attestation_sha256"] = protected.canonical_json_fingerprint( + {key: value for key, value in attestation.items() if key != "attestation_sha256"} + ) + + report = protected.build_admission_report(attestation=attestation, now=NOW) + + assert report["admission_eligible"] is False + assert "protected admission attestation is outside the freshness window" in report[ + "attestation_errors" + ] + + +def test_requirement_path_marker_and_missing_check_fail_closed(): + attestation = deepcopy(_attestation()) + attestation["requirements"]["owner_decision"]["attestation_sha256"] = "/private/owner.json" + attestation["checks"].pop("reviewer_approval") + attestation["attestation_sha256"] = protected.canonical_json_fingerprint( + {key: value for key, value in attestation.items() if key != "attestation_sha256"} + ) + + report = protected.build_admission_report(attestation=attestation, now=NOW) + + assert report["admission_eligible"] is False + assert "protected admission attestation contains a path or payload marker" in report[ + "attestation_errors" + ] + assert "protected admission check inventory does not match" in report[ + "attestation_errors" + ] + + +def test_report_tampering_cannot_create_authority(): + report = protected.build_admission_report(attestation=_attestation(), now=NOW) + report["content_admission_authorized"] = True + + assert "protected admission report fingerprint does not match" in ( + protected.verify_report_integrity(report) + ) + + +def test_attestation_lifetime_is_bounded(): + attestation = _attestation() + attestation["expires_at"] = (NOW + timedelta(days=8)).isoformat() + attestation["attestation_sha256"] = protected.canonical_json_fingerprint( + {key: value for key, value in attestation.items() if key != "attestation_sha256"} + ) + + report = protected.build_admission_report(attestation=attestation, now=NOW) + + assert "protected admission attestation lifetime exceeds seven days" in report[ + "attestation_errors" + ] diff --git a/docs/architecture-decisions/adr-079-chongqing-protected-attestation-intake.md b/docs/architecture-decisions/adr-079-chongqing-protected-attestation-intake.md new file mode 100644 index 00000000..102d04dc --- /dev/null +++ b/docs/architecture-decisions/adr-079-chongqing-protected-attestation-intake.md @@ -0,0 +1,70 @@ +# ADR-079: Chongqing protected admission attestation intake + +**Status**: Accepted + +**Date**: 2026-08-17 + +**Decision owners**: Platform Architecture, Data Platform, Data Governance, Security + +## Context + +M3-31 provides one immutable, metadata-only readiness record for the first +Chongqing land-parcel candidate. It intentionally leaves six derivation inputs, +eight governance decisions and one fresh protected attestation unresolved. The +next workflow needs a deterministic intake boundary so that external evidence +can be checked without copying source payloads or allowing the evaluator to +become an ingestion authority. + +## Decision + +Adopt `data_agent.chongqing_protected_admission` as the M3-32 read-only intake +and evaluation contract. An external attestation bundle must: + +1. bind to the exact M3-31 logical evidence fingerprint and file SHA-256; +2. repeat the source identity binding without source records, geometry or local + paths; +3. provide all fifteen requirement records as independently attested + `verified` entries with SHA-256 fingerprints; +4. pass the fixed archive, extraction, governance, binding, privacy and + no-mutation check inventory; +5. identify the protected verifier and a non-local HTTPS evidence URI; and +6. be observed within 24 hours, unexpired, and valid for no more than seven + days. + +The evaluator produces a fingerprinted report. A complete external bundle may +set `admission_eligible=true` in that report, meaning the evidence is ready for +the separate admission decision boundary. It never sets content admission, +Landing, ResourceVersion, PlatformRun, scheduler submission, provider mutation, +or production readiness authority to true. + +## Authority boundary + +`validate` checks the checked-in M3-31 baseline and succeeds while the baseline +is valid but blocked. `evaluate` requires an external attestation file and +fails closed on any drift, missing requirement, stale timestamp, path/payload +marker, or failed check. `verify` rejects report tampering and any authority or +production overclaim. No command reads or copies the Chongqing source payload. + +Synthetic complete attestation fixtures in unit tests exercise the evaluator +only; they are not production evidence and are not checked in as an admission +decision. + +## Consequences + +**Positive**: the next protected workflow has a stable, testable input schema +and can report exactly which of the fifteen external requirements remain +blocked. + +**Positive**: complete evidence cannot be confused with write authority; the +report retains an explicit no-mutation and no-production boundary. + +**Negative**: AR-2 remains `in_progress` until real external attestations are +provided and reviewed by the protected verifier. The checked baseline remains +`admission_eligible=false` because no attestation is present. + +## Verification + +```bash +./scripts/chongqing-protected-admission.sh validate +python -m pytest data_agent/test_chongqing_protected_admission.py -q +``` diff --git a/docs/roadmap.md b/docs/roadmap.md index 3533e98f..41413ca0 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -443,6 +443,7 @@ AR-0 Architecture / Schema / Runtime Truth - M3-29 extraction provenance contract：以 M3-28 上游 evidence fingerprint、archive/extracted comparison 和明确的 derivation blocker 固定“已观察比较”与“派生证明缺失”的边界；operator、tool、command、modified/additional manifest 和 archive-to-working-set attestation 未齐全前不得 content admission。 - M3-30 source governance gate：选择 `bishan_land_use_dltb_local` 作为首条地类图斑候选，绑定 M3-28/M3-29 指纹并把 owner、license、retention、access、privacy/sensitivity、标准版本、DataSLO、golden result 八项决策冻结为独立 pending records；完整派生证明、签名决策和 fresh protected attestation 未齐全前不得 content admission。 - M3-31 protected admission readiness contract：绑定 M3-28/M3-29/M3-30 的逻辑与文件 fingerprints，把 6 项 derivation、8 项 governance 和 1 项 protected attestation 固定为 15 个 fail-closed requirements；`admission_eligible=false` 前不得创建 Landing、ResourceVersion、PlatformRun、scheduler submission 或 provider mutation。 +- M3-32 protected admission attestation intake：固定外部 attestation 的 M3-31 logical/file fingerprint binding、15 项逐项 SHA-256 证明、受保护 verifier/source binding、24 小时 freshness、七天 validity 上限和八项 no-payload/no-mutation checks；`evaluate` 只产生 fingerprinted readiness report，不创建任何 Landing、ResourceVersion、PlatformRun、scheduler submission 或 provider mutation authority。 - SourceDefinition、CredentialReference、SourceCapability、SyncDefinition/Version、SyncRun、Cursor/Watermark、SchemaDriftEvent 和 Reconciliation。 - 数据库、对象存储/空间文件、HTTP/STAC 三类代表 source 的连接、凭据、连通、发现、preview、profile 和 owner 登记。 - 全量/增量微批的 Append/Overwrite/Merge 策略，以及至少一个真实 CDC 或事件流 source 通过 Flink 写入版本化 Bronze；覆盖 watermark/offset、checkpoint、迟到/乱序、源端删除、幂等、对账、重放和失败恢复。 @@ -690,7 +691,7 @@ Golden checks 至少覆盖： 3. 冻结 ResourceURN、ResourceVersion、PlatformDefinition/PlatformRun/FrameworkAttemptObservation/Artifact/LineageEvent、SubjectContext 与 storage/table/compute provider 最小合同。 4. 分阶段实现 `gda-metadata-fabric-bridge`：M1 只读 mapping/reconciliation、M2a 本地 foundation/重启连续性、M2b-1 本地三存储恢复、M2b-2 隔离 versioned/Object-Locked repository round-trip、M2b-3 本机双集群 + Kubernetes 外 COMPLIANCE repository + 独立 writer/reader、M2c-1 provider-native metrics、M2c-2 临时 OTel Collector + JSON Exporter 的双周期本地 pipeline、M2c-3 本地单 job scrape 故障检测/配置恢复/完整清理 evidence、M2c-4 绑定 source revision 的 production observability readiness contract，以及 M2d-1 本地 kindnet 跨节点 NetworkPolicy enforcement 已验证；M2c-4 当前仍有 20 项 blockers，M2d-1 也未验证生产 provider policy 或 tenant isolation。下一步完成 source host/cluster 外的生产 bucket、KMS/TLS/workload identity、source-loss recovery 与 RPO/RTO，并批准 metrics backend、retention、OTel/TLS、tenant、alert/SLO/owner 后在受保护环境验证持续采集、存储、查询、真实告警投递、runbook 响应和 provider NetworkPolicy；再推进 OIDC、upgrade/rollback、registry provenance 和 owner/runbook；之后才进入 M3 ingestion/OpenLineage/conformance。 5. 实现 `gda-orchestration-gateway`、DolphinScheduler process/task/schedule/complement/worker-group、Spark/Flink provider task adapter 和故障注入；不再开发新的 lease/queue/scheduler。 -6. M3-28 已冻结全量重庆真实源的 path-free physical/metadata admission baseline，M3-29 已建立 metadata-only extraction provenance gap baseline，M3-30 已选择 `bishan_land_use_dltb_local` 并把八项治理输入固化为 fail-closed pending records，M3-31 已将三层证据统一成 protected admission readiness contract；下一步取得 15 项外部 attestation 并由受保护 verifier 重新评估，未获批准前不得 content admission。 +6. M3-28 已冻结全量重庆真实源的 path-free physical/metadata admission baseline，M3-29 已建立 metadata-only extraction provenance gap baseline，M3-30 已选择 `bishan_land_use_dltb_local` 并把八项治理输入固化为 fail-closed pending records，M3-31 已将三层证据统一成 protected admission readiness contract，M3-32 已固定 protected attestation intake/evaluate/verify boundary；下一步取得 15 项外部 attestation 并由受保护 verifier 重新评估，未获批准前不得 content admission。 7. 冻结 Default Lakehouse、Cloud Managed、Lightweight Integrated profiles；以统一 Run 完成默认 MinIO/Iceberg/Spark/Flink、轻量 PostGIS/DuckDB 和 Azure 代表 adapter 的 conformance smoke。 8. 实现跨 profile 的 Raw -> ODS -> DIM/DWD -> DWS -> ADS 通用生产、质量、发布、回滚和 golden equivalence。 9. 建立 DataProductBlueprint、模型版本和 Visual/SQL/Notebook 共用 definition 的 Build 工作台，打通 preview、test、publish、approval 和 rollback。 @@ -731,7 +732,7 @@ AR-4 parity/control gate 退出前暂停以下主线扩张： |---|---|---| | AR-0 Architecture/Schema/Runtime Truth Freeze | `in_progress` | 全环境 schema/config fingerprint、迁移 fail-closed、事实清单、storage/compute/GIS serving provider profile/capability、ADR-017 benchmark、owner/SLO 和首条数据/服务验收集冻结 | | AR-1 Unified Metadata + Orchestration Control Planes | `in_progress` | controlled gateway、DolphinScheduler adapter、Metadata Fabric M1/M2a/M2b、M2c-1 provider metrics、M2c-2 本地临时 OTel pipeline、M2c-3 本地 scrape failure/recovery、M2c-4 production observability readiness contract 与 M2d-1 本地跨节点 NetworkPolicy enforcement 已验证，生产观测和生产 policy/tenant isolation 仍 blocked；下一证据是 source host/cluster 外的生产 recovery、持久 metrics backend/TLS/tenant/真实 alert delivery/SLO、OIDC、受保护 provider NetworkPolicy、升级回滚/registry provenance，以及受控 ingestion/replay 与无双写验收 | -| AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `in_progress` | M3-28 admission baseline、M3-29 extraction provenance gap baseline、M3-30 first-candidate governance gate 与 M3-31 protected admission readiness contract 已完成；下一证据是 15 项外部 attestation、受保护 verifier 复核，随后才可授权 immutable Landing authority、content admission 和 ingestion；最终仍需三类代表源、`DriveTransfer` 云盘客户端、大文件恢复、默认湖仓、轻量存算一体及 Azure 代表 adapter 通过 provider conformance 与 Raw -> ADS 验收 | +| AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `in_progress` | M3-28 admission baseline、M3-29 extraction provenance gap baseline、M3-30 first-candidate governance gate、M3-31 protected admission readiness contract 与 M3-32 protected attestation intake 已完成；checked baseline 仍为 `admission_eligible=false`，下一证据是 15 项外部 attestation、受保护 verifier 复核，随后才可授权 immutable Landing authority、content admission 和 ingestion；最终仍需三类代表源、`DriveTransfer` 云盘客户端、大文件恢复、默认湖仓、轻量存算一体及 Azure 代表 adapter 通过 provider conformance 与 Raw -> ADS 验收 | | AR-3 Data Product Engineering + Governance Workbench | `planned` | Blueprint、模型、Visual/SQL/Notebook、DataOps CI/CD、质量/安全/审批共用 definition 和产品生命周期 | | AR-4 Asset/GIS Service/Spatial Experience Operations | `planned` | Service Control Plane、Features/Tiles/MVT/COG/STAC/export 及条件 legacy OGC/3D/EDR provider、Gateway/权限/缓存、原子切换/回滚、Discover/Operate/Govern 和无 LLM 多入口通过 conformance/parity/control gate | | AR-5 AgentOps Runtime + UX Uplift | `planned` | DataOps parity/control 通过；Agent bundle eval、deployment、online observation、incident/rollback 和 uplift gate | diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index 853ce420..d0473b3c 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -66,6 +66,7 @@ 20. M3-29 只允许记录 M3-28 上游 fingerprint 和 archive/extracted comparison；`comparison_observed=true`、`derivation_provenance_complete=false`，operator/tool/command、modified/additional manifest 和 archive-to-working-set attestation 缺失时，不形成 Landing object、ResourceVersion、PlatformRun、scheduler submission、provider mutation、content admission 或 production ingestion 权威；provenance evidence 不得被 rehash 成批准。 21. M3-30 只允许选择 `bishan_land_use_dltb_local` 作为首条候选并记录 owner、license、retention、access、privacy/sensitivity、标准版本、DataSLO、golden result 八项 pending decisions；`candidate_scope_selected=true` 不等于 `source_governance_approved`，checked governance evidence 不形成 Landing object、ResourceVersion、PlatformRun、scheduler submission、provider mutation、content admission 或 production ingestion 权威，也不得被 rehash 成批准。 22. M3-31 只允许绑定 M3-28/M3-29/M3-30 并记录 6 项 derivation、8 项 governance 与 1 项 protected attestation 的 pending requirements；`admission_eligible=false` 时不形成 Landing object、ResourceVersion、PlatformRun、scheduler submission、provider mutation、content admission 或 production ingestion 权威，readiness evidence 不得被 rehash 成批准。 +23. M3-32 只允许受保护 verifier 消费绑定 M3-31 logical/file fingerprints 的外部 attestation；15 项 requirement、八项 no-payload/no-mutation checks、freshness、expiry 和 verifier/source binding 任一缺失即 blocked。完整 attestation 只能使 fingerprinted report 的 `admission_eligible=true`，不创建 Landing object、ResourceVersion、PlatformRun、scheduler submission、provider mutation、content admission 或 production readiness 权威；合成测试 attestation 不计入生产证据。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -101,6 +102,7 @@ - AR-2 M3-29 已建立重庆 extraction provenance gap baseline：evidence SHA 为 `b56ce0c036827d4338ab2cfae8f3fb4c9e1e78ec18aac243272f1f77801300ef`，evidence file SHA 为 `cfae0478c76452a155e8af42ec8499e4e7876a49c1dbb98648526025cb154360`；上游 M3-28 evidence/file SHA 分别为 `a2196495d845d61be939c7fc36a7f05c3567e365599d2d04be0aab9c568459c1` / `9b5c20369c235f7e0a2f2cb0a21cee77f86981aa273bac196605a4803b05ce83`；比较固定为 526 exact、6 modified、0 missing、52 additional，6 项 derivation evidence 缺失，`comparison_observed=true`、`derivation_provenance_complete=false`、`source_content_admitted=false`、`production_ready=false`。该证据只记录比较关系，不含 source payload、绝对路径、记录值或 geometry，也不证明 archive-to-working-set attestation、Landing authority、ResourceVersion、Run、scheduler/provider mutation 或生产 ingestion。 - AR-2 M3-30 已建立首条重庆地类图斑 source governance gate：候选为 `bishan_land_use_dltb_local`，evidence SHA 为 `97cf11ab8938c048dce9db903d1a4f30758f208dec6dad1a08b740a4a8fe7b6f`，evidence file SHA 为 `25bc5e2dfc5528f5556e7174f8c99fed7abaf30b9312528f5164c16bdf7cca9a`；证据绑定 M3-28/M3-29 fingerprints，八项治理 decision records 全部为 `pending`，另保留 derivation 与 fresh protected attestation blockers；`candidate_scope_selected=true`、`source_governance_approved=false`、`source_content_admitted=false`、`production_ready=false`。该证据不含 source payload、绝对路径、记录值或 geometry，也不证明任何治理批准、Landing authority、ResourceVersion、Run、scheduler/provider mutation 或生产 ingestion。 - AR-2 M3-31 已建立 protected admission readiness contract：候选为 `bishan_land_use_dltb_local`，evidence SHA 为 `2f5ae24ab904af0eed18ee7c517ab5c4638cbdf0923c9345b0041af185d25591`，evidence file SHA 为 `c595065e152988529ff12e2301d59caebb31d2889658a676c9d1f8239e6f8372`；证据绑定 M3-28/M3-29/M3-30 的逻辑与文件 fingerprints，15 项要求全部为 `missing`，`admission_eligible=false`、`source_content_admitted=false`、`production_ready=false`。该证据只定义未来受保护 verifier 的输入，不含 source payload、绝对路径、记录值或 geometry，也不证明任何 Landing authority、ResourceVersion、Run、scheduler/provider mutation 或生产 ingestion。 +- AR-2 M3-32 已建立 protected admission attestation intake/evaluate/verify contract：外部输入必须绑定 M3-31 logical/file fingerprints，逐项提供 15 个 SHA-256 attestation、八项 protected checks、verifier/source binding 和受控 freshness/expiry；checked baseline 的 `readiness_valid=true`、`attestation_valid=false`、`admission_eligible=false`，所有 content/authority/production claims 继续为 `false`。合成完整 attestation 仅覆盖 evaluator 单元测试，不计入真实准入证据。 ## 下一验收证据 @@ -109,5 +111,5 @@ - 为受保护 activation 提供真实 ConfigMap snapshot、Secret key attestation、provider identity 和 reviewer approval，随后完成 managed outbox worker/provider callback 单副本实际部署、唯一 worker ID、status/lease 故障恢复和无双写证据； - 首条真实图斑链对 golden slice 的 output hash、独立质量结果/evidence、血缘、发布 revision 和 rollback 演练； - OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery、M3-5/M3-6 本地临时 provider identity 与 M3-7 pending profile/合成 attestation 均不计入生产退出门； -- M3-29/M3-30/M3-31 的下一证据必须补齐 operator/tool/command、modified/additional manifest、archive-to-working-set attestation、owner/license/retention/access/privacy-sensitivity/standard-version/DataSLO/golden-result 八项签名决策与 fresh protected attestation，并由受保护 verifier 重新计算 admission eligibility；metadata-only admission/provenance/governance/readiness evidence 是不可变检查结果，不能被编辑或重解释为内容批准、Landing authority 或 ingestion authorization； +- M3-29/M3-30/M3-31/M3-32 的下一证据必须补齐 operator/tool/command、modified/additional manifest、archive-to-working-set attestation、owner/license/retention/access/privacy-sensitivity/standard-version/DataSLO/golden-result 八项签名决策与 fresh protected attestation，并由受保护 verifier 按 M3-32 intake contract 重新计算 admission eligibility；metadata-only admission/provenance/governance/readiness/evaluation evidence 是不可变检查结果，不能被编辑或重解释为内容批准、Landing authority 或 ingestion authorization； - DolphinScheduler/Temporal sandbox 的独立数据库、备份恢复、身份、版本和升级责任证明；DolphinScheduler standalone/H2 不计入此退出门。 diff --git a/scripts/chongqing-protected-admission.sh b/scripts/chongqing-protected-admission.sh new file mode 100755 index 00000000..7b97a832 --- /dev/null +++ b/scripts/chongqing-protected-admission.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +common_git_dir="$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +shared_root="" +if [ -n "$common_git_dir" ]; then + shared_root="$(cd "$common_git_dir/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$repo_root/.venv/bin/python" ]; then + PYTHON="$repo_root/.venv/bin/python" +elif [ -n "$shared_root" ] && [ -x "$shared_root/.venv/bin/python" ]; then + PYTHON="$shared_root/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$repo_root" +exec "$PYTHON" -m data_agent.chongqing_protected_admission "$@"