Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/cd-staging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ jobs:
data_agent/test_chongqing_extraction_provenance.py \
data_agent/test_chongqing_source_governance.py \
data_agent/test_chongqing_admission_readiness.py \
data_agent/test_chongqing_protected_admission.py \
data_agent/test_dolphinscheduler_adapter.py \
data_agent/test_dolphinscheduler_command_consumer.py \
data_agent/test_dolphinscheduler_command_worker.py \
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@ jobs:
- name: Validate Chongqing admission readiness evidence
run: python -m data_agent.chongqing_admission_readiness

- name: Validate Chongqing protected admission intake
run: python -m data_agent.chongqing_protected_admission validate

- name: Validate DolphinScheduler adapter boundary
run: python -m data_agent.dolphinscheduler_adapter validate

Expand Down Expand Up @@ -175,6 +178,7 @@ jobs:
data_agent/test_chongqing_extraction_provenance.py \
data_agent/test_chongqing_source_governance.py \
data_agent/test_chongqing_admission_readiness.py \
data_agent/test_chongqing_protected_admission.py \
data_agent/test_dolphinscheduler_adapter.py \
data_agent/test_dolphinscheduler_command_consumer.py \
data_agent/test_dolphinscheduler_command_worker.py \
Expand Down
384 changes: 384 additions & 0 deletions data_agent/chongqing_protected_admission.py

Large diffs are not rendered by default.

141 changes: 141 additions & 0 deletions data_agent/test_chongqing_protected_admission.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import json
from copy import deepcopy
from datetime import UTC, datetime, timedelta

from data_agent import chongqing_admission_readiness as readiness
from data_agent import chongqing_protected_admission as protected

NOW = datetime(2026, 8, 17, 12, 0, tzinfo=UTC)


def _readiness() -> dict:
return json.loads(readiness.DEFAULT_EVIDENCE_PATH.read_text(encoding="utf-8"))


def _attestation() -> dict:
evidence = _readiness()
source = evidence["source_binding"]
value = {
"schema": protected.ATTESTATION_SCHEMA,
"readiness_evidence_sha256": evidence["evidence_sha256"],
"readiness_evidence_file_sha256": readiness.EVIDENCE_FILE_SHA256,
"source_binding": {
key: source[key]
for key in ("source_id", "source_group_id", "asset_id", "source_ref")
},
"observed_at": "2026-08-17T11:45:00Z",
"expires_at": "2026-08-18T11:45:00Z",
"protected_environment": protected.PROTECTED_ENVIRONMENT,
"verifier_identity": "protected-admission-verifier@platform",
"evidence_uri": "https://attestations.gisplatform.com/admission/2026-08-17",
"requirements": {
key: {"status": "verified", "attestation_sha256": "a" * 64}
for key in readiness.REQUIREMENT_KEYS
},
"checks": {key: "passed" for key in protected.EXPECTED_CHECKS},
}
stable = dict(value)
value["attestation_sha256"] = protected.canonical_json_fingerprint(stable)
return value


def test_pending_profile_is_valid_but_protected_attestation_is_missing():
report = protected.build_admission_report(now=NOW)

assert report["readiness_valid"] is True
assert report["attestation_valid"] is False
assert report["admission_eligible"] is False
assert report["attestation_errors"] == [
"protected admission attestation is missing"
]
assert report["content_admission_authorized"] is False
assert report["production_ready"] is False


def test_complete_attestation_only_passes_the_readiness_evaluation():
report = protected.build_admission_report(attestation=_attestation(), now=NOW)

assert report["readiness_valid"] is True
assert report["attestation_valid"] is True
assert report["admission_eligible"] is True
assert report["content_admission_authorized"] is False
assert report["source_content_admitted"] is False
assert report["landing_authority_created"] is False
assert report["resource_version_created"] is False
assert report["platform_run_created"] is False
assert report["scheduler_submission_authorized"] is False
assert report["provider_mutation_authorized"] is False
assert report["production_ready"] is False
assert protected.verify_report_integrity(report) == []


def test_attestation_binding_drift_fails_closed():
attestation = _attestation()
attestation["source_binding"]["asset_id"] = "other_asset"
attestation["attestation_sha256"] = protected.canonical_json_fingerprint(
{key: value for key, value in attestation.items() if key != "attestation_sha256"}
)

report = protected.build_admission_report(attestation=attestation, now=NOW)

assert report["admission_eligible"] is False
assert "protected admission source binding does not match M3-31" in report[
"attestation_errors"
]


def test_stale_attestation_fails_closed():
attestation = _attestation()
attestation["observed_at"] = "2026-08-15T11:45:00Z"
attestation["attestation_sha256"] = protected.canonical_json_fingerprint(
{key: value for key, value in attestation.items() if key != "attestation_sha256"}
)

report = protected.build_admission_report(attestation=attestation, now=NOW)

assert report["admission_eligible"] is False
assert "protected admission attestation is outside the freshness window" in report[
"attestation_errors"
]


def test_requirement_path_marker_and_missing_check_fail_closed():
attestation = deepcopy(_attestation())
attestation["requirements"]["owner_decision"]["attestation_sha256"] = "/private/owner.json"
attestation["checks"].pop("reviewer_approval")
attestation["attestation_sha256"] = protected.canonical_json_fingerprint(
{key: value for key, value in attestation.items() if key != "attestation_sha256"}
)

report = protected.build_admission_report(attestation=attestation, now=NOW)

assert report["admission_eligible"] is False
assert "protected admission attestation contains a path or payload marker" in report[
"attestation_errors"
]
assert "protected admission check inventory does not match" in report[
"attestation_errors"
]


def test_report_tampering_cannot_create_authority():
report = protected.build_admission_report(attestation=_attestation(), now=NOW)
report["content_admission_authorized"] = True

assert "protected admission report fingerprint does not match" in (
protected.verify_report_integrity(report)
)


def test_attestation_lifetime_is_bounded():
attestation = _attestation()
attestation["expires_at"] = (NOW + timedelta(days=8)).isoformat()
attestation["attestation_sha256"] = protected.canonical_json_fingerprint(
{key: value for key, value in attestation.items() if key != "attestation_sha256"}
)

report = protected.build_admission_report(attestation=attestation, now=NOW)

assert "protected admission attestation lifetime exceeds seven days" in report[
"attestation_errors"
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# ADR-079: Chongqing protected admission attestation intake

**Status**: Accepted

**Date**: 2026-08-17

**Decision owners**: Platform Architecture, Data Platform, Data Governance, Security

## Context

M3-31 provides one immutable, metadata-only readiness record for the first
Chongqing land-parcel candidate. It intentionally leaves six derivation inputs,
eight governance decisions and one fresh protected attestation unresolved. The
next workflow needs a deterministic intake boundary so that external evidence
can be checked without copying source payloads or allowing the evaluator to
become an ingestion authority.

## Decision

Adopt `data_agent.chongqing_protected_admission` as the M3-32 read-only intake
and evaluation contract. An external attestation bundle must:

1. bind to the exact M3-31 logical evidence fingerprint and file SHA-256;
2. repeat the source identity binding without source records, geometry or local
paths;
3. provide all fifteen requirement records as independently attested
`verified` entries with SHA-256 fingerprints;
4. pass the fixed archive, extraction, governance, binding, privacy and
no-mutation check inventory;
5. identify the protected verifier and a non-local HTTPS evidence URI; and
6. be observed within 24 hours, unexpired, and valid for no more than seven
days.

The evaluator produces a fingerprinted report. A complete external bundle may
set `admission_eligible=true` in that report, meaning the evidence is ready for
the separate admission decision boundary. It never sets content admission,
Landing, ResourceVersion, PlatformRun, scheduler submission, provider mutation,
or production readiness authority to true.

## Authority boundary

`validate` checks the checked-in M3-31 baseline and succeeds while the baseline
is valid but blocked. `evaluate` requires an external attestation file and
fails closed on any drift, missing requirement, stale timestamp, path/payload
marker, or failed check. `verify` rejects report tampering and any authority or
production overclaim. No command reads or copies the Chongqing source payload.

Synthetic complete attestation fixtures in unit tests exercise the evaluator
only; they are not production evidence and are not checked in as an admission
decision.

## Consequences

**Positive**: the next protected workflow has a stable, testable input schema
and can report exactly which of the fifteen external requirements remain
blocked.

**Positive**: complete evidence cannot be confused with write authority; the
report retains an explicit no-mutation and no-production boundary.

**Negative**: AR-2 remains `in_progress` until real external attestations are
provided and reviewed by the protected verifier. The checked baseline remains
`admission_eligible=false` because no attestation is present.

## Verification

```bash
./scripts/chongqing-protected-admission.sh validate
python -m pytest data_agent/test_chongqing_protected_admission.py -q
```
5 changes: 3 additions & 2 deletions docs/roadmap.md
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,7 @@ AR-0 Architecture / Schema / Runtime Truth
- M3-29 extraction provenance contract:以 M3-28 上游 evidence fingerprint、archive/extracted comparison 和明确的 derivation blocker 固定“已观察比较”与“派生证明缺失”的边界;operator、tool、command、modified/additional manifest 和 archive-to-working-set attestation 未齐全前不得 content admission。
- M3-30 source governance gate:选择 `bishan_land_use_dltb_local` 作为首条地类图斑候选,绑定 M3-28/M3-29 指纹并把 owner、license、retention、access、privacy/sensitivity、标准版本、DataSLO、golden result 八项决策冻结为独立 pending records;完整派生证明、签名决策和 fresh protected attestation 未齐全前不得 content admission。
- M3-31 protected admission readiness contract:绑定 M3-28/M3-29/M3-30 的逻辑与文件 fingerprints,把 6 项 derivation、8 项 governance 和 1 项 protected attestation 固定为 15 个 fail-closed requirements;`admission_eligible=false` 前不得创建 Landing、ResourceVersion、PlatformRun、scheduler submission 或 provider mutation。
- M3-32 protected admission attestation intake:固定外部 attestation 的 M3-31 logical/file fingerprint binding、15 项逐项 SHA-256 证明、受保护 verifier/source binding、24 小时 freshness、七天 validity 上限和八项 no-payload/no-mutation checks;`evaluate` 只产生 fingerprinted readiness report,不创建任何 Landing、ResourceVersion、PlatformRun、scheduler submission 或 provider mutation authority。
- SourceDefinition、CredentialReference、SourceCapability、SyncDefinition/Version、SyncRun、Cursor/Watermark、SchemaDriftEvent 和 Reconciliation。
- 数据库、对象存储/空间文件、HTTP/STAC 三类代表 source 的连接、凭据、连通、发现、preview、profile 和 owner 登记。
- 全量/增量微批的 Append/Overwrite/Merge 策略,以及至少一个真实 CDC 或事件流 source 通过 Flink 写入版本化 Bronze;覆盖 watermark/offset、checkpoint、迟到/乱序、源端删除、幂等、对账、重放和失败恢复。
Expand Down Expand Up @@ -690,7 +691,7 @@ Golden checks 至少覆盖:
3. 冻结 ResourceURN、ResourceVersion、PlatformDefinition/PlatformRun/FrameworkAttemptObservation/Artifact/LineageEvent、SubjectContext 与 storage/table/compute provider 最小合同。
4. 分阶段实现 `gda-metadata-fabric-bridge`:M1 只读 mapping/reconciliation、M2a 本地 foundation/重启连续性、M2b-1 本地三存储恢复、M2b-2 隔离 versioned/Object-Locked repository round-trip、M2b-3 本机双集群 + Kubernetes 外 COMPLIANCE repository + 独立 writer/reader、M2c-1 provider-native metrics、M2c-2 临时 OTel Collector + JSON Exporter 的双周期本地 pipeline、M2c-3 本地单 job scrape 故障检测/配置恢复/完整清理 evidence、M2c-4 绑定 source revision 的 production observability readiness contract,以及 M2d-1 本地 kindnet 跨节点 NetworkPolicy enforcement 已验证;M2c-4 当前仍有 20 项 blockers,M2d-1 也未验证生产 provider policy 或 tenant isolation。下一步完成 source host/cluster 外的生产 bucket、KMS/TLS/workload identity、source-loss recovery 与 RPO/RTO,并批准 metrics backend、retention、OTel/TLS、tenant、alert/SLO/owner 后在受保护环境验证持续采集、存储、查询、真实告警投递、runbook 响应和 provider NetworkPolicy;再推进 OIDC、upgrade/rollback、registry provenance 和 owner/runbook;之后才进入 M3 ingestion/OpenLineage/conformance。
5. 实现 `gda-orchestration-gateway`、DolphinScheduler process/task/schedule/complement/worker-group、Spark/Flink provider task adapter 和故障注入;不再开发新的 lease/queue/scheduler。
6. M3-28 已冻结全量重庆真实源的 path-free physical/metadata admission baseline,M3-29 已建立 metadata-only extraction provenance gap baseline,M3-30 已选择 `bishan_land_use_dltb_local` 并把八项治理输入固化为 fail-closed pending records,M3-31 已将三层证据统一成 protected admission readiness contract;下一步取得 15 项外部 attestation 并由受保护 verifier 重新评估,未获批准前不得 content admission。
6. M3-28 已冻结全量重庆真实源的 path-free physical/metadata admission baseline,M3-29 已建立 metadata-only extraction provenance gap baseline,M3-30 已选择 `bishan_land_use_dltb_local` 并把八项治理输入固化为 fail-closed pending records,M3-31 已将三层证据统一成 protected admission readiness contract,M3-32 已固定 protected attestation intake/evaluate/verify boundary;下一步取得 15 项外部 attestation 并由受保护 verifier 重新评估,未获批准前不得 content admission。
7. 冻结 Default Lakehouse、Cloud Managed、Lightweight Integrated profiles;以统一 Run 完成默认 MinIO/Iceberg/Spark/Flink、轻量 PostGIS/DuckDB 和 Azure 代表 adapter 的 conformance smoke。
8. 实现跨 profile 的 Raw -> ODS -> DIM/DWD -> DWS -> ADS 通用生产、质量、发布、回滚和 golden equivalence。
9. 建立 DataProductBlueprint、模型版本和 Visual/SQL/Notebook 共用 definition 的 Build 工作台,打通 preview、test、publish、approval 和 rollback。
Expand Down Expand Up @@ -731,7 +732,7 @@ AR-4 parity/control gate 退出前暂停以下主线扩张:
|---|---|---|
| AR-0 Architecture/Schema/Runtime Truth Freeze | `in_progress` | 全环境 schema/config fingerprint、迁移 fail-closed、事实清单、storage/compute/GIS serving provider profile/capability、ADR-017 benchmark、owner/SLO 和首条数据/服务验收集冻结 |
| AR-1 Unified Metadata + Orchestration Control Planes | `in_progress` | controlled gateway、DolphinScheduler adapter、Metadata Fabric M1/M2a/M2b、M2c-1 provider metrics、M2c-2 本地临时 OTel pipeline、M2c-3 本地 scrape failure/recovery、M2c-4 production observability readiness contract 与 M2d-1 本地跨节点 NetworkPolicy enforcement 已验证,生产观测和生产 policy/tenant isolation 仍 blocked;下一证据是 source host/cluster 外的生产 recovery、持久 metrics backend/TLS/tenant/真实 alert delivery/SLO、OIDC、受保护 provider NetworkPolicy、升级回滚/registry provenance,以及受控 ingestion/replay 与无双写验收 |
| AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `in_progress` | M3-28 admission baseline、M3-29 extraction provenance gap baseline、M3-30 first-candidate governance gateM3-31 protected admission readiness contract 已完成;下一证据是 15 项外部 attestation、受保护 verifier 复核,随后才可授权 immutable Landing authority、content admission 和 ingestion;最终仍需三类代表源、`DriveTransfer` 云盘客户端、大文件恢复、默认湖仓、轻量存算一体及 Azure 代表 adapter 通过 provider conformance 与 Raw -> ADS 验收 |
| AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `in_progress` | M3-28 admission baseline、M3-29 extraction provenance gap baseline、M3-30 first-candidate governance gateM3-31 protected admission readiness contract 与 M3-32 protected attestation intake 已完成;checked baseline 仍为 `admission_eligible=false`,下一证据是 15 项外部 attestation、受保护 verifier 复核,随后才可授权 immutable Landing authority、content admission 和 ingestion;最终仍需三类代表源、`DriveTransfer` 云盘客户端、大文件恢复、默认湖仓、轻量存算一体及 Azure 代表 adapter 通过 provider conformance 与 Raw -> ADS 验收 |
| AR-3 Data Product Engineering + Governance Workbench | `planned` | Blueprint、模型、Visual/SQL/Notebook、DataOps CI/CD、质量/安全/审批共用 definition 和产品生命周期 |
| AR-4 Asset/GIS Service/Spatial Experience Operations | `planned` | Service Control Plane、Features/Tiles/MVT/COG/STAC/export 及条件 legacy OGC/3D/EDR provider、Gateway/权限/缓存、原子切换/回滚、Discover/Operate/Govern 和无 LLM 多入口通过 conformance/parity/control gate |
| AR-5 AgentOps Runtime + UX Uplift | `planned` | DataOps parity/control 通过;Agent bundle eval、deployment、online observation、incident/rollback 和 uplift gate |
Expand Down
Loading
Loading