diff --git a/.env.example b/.env.example index 7503553..c79b8cf 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,8 @@ WORKER_DISPATCH_ENABLED=0 WORKER_IMAGE_VALIDATION_ENABLED=0 WORKER_INFERENCE_ENABLED=0 WORKER_RULE_EVALUATION_ENABLED=0 +# CSV report publisher/consumer opt-in; reports use q.reports independently of q.general. +WORKER_REPORT_EXPORT_ENABLED=0 # Identity, foundation and image-job query APIs are opt-in; use this exact HTTPS origin. API_IDENTITY_ENABLED=0 PUBLIC_ORIGIN=https://labsafe.example.org @@ -29,6 +31,7 @@ S3_REGION=us-east-1 S3_ACCESS_KEY_FILE=.local-secrets/s3-api-access-key S3_SECRET_KEY_FILE=.local-secrets/s3-api-secret-key # General Worker credentials: staging GET/version; original/analysis GET/version + PUT. +# CSV reports also need HEAD/GET/version + PUT on authorized reports (R) paths. # No Delete, ListBucket or administration; never fall back to API credentials. S3_WORKER_ACCESS_KEY_FILE=.local-secrets/s3-worker-access-key S3_WORKER_SECRET_KEY_FILE=.local-secrets/s3-worker-secret-key diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index aeee9fe..5592e74 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -19,7 +19,7 @@ I-02F2 新增 completeUpload/getImage；上传完成受理为 validating，并 I-03A1/A2 接通上传验证任务持久调度、租约、围栏及恢复；I-02F3 接通可选 general Worker、真实 Pillow 图像处理与 ready/rejected 事务。 I-03A3 新增 getJob/listDeadLetters/replayJob 的 validate_image 分支，累计注册 34 个公共业务 API；管理员重放保留固定输入和历史。 I-02F4 新增 downloadImage，累计注册 35 个公共业务 API；ready 分析图按实验室 READ 下载，原图额外 Admin 与审计；签名固定版本、60 秒。 -尚不包含 crop/报告下载、完整巡检业务 API、其他任务类型/事件 inbox/AI 闭环、训练权重或生产部署；真实 MinIO/IAM/HTTPS 验收仍待完成。 +已实现整改与CSV报告受理/执行/下载；尚不包含 crop 下载、完整巡检业务 API、其他任务类型/事件 inbox/AI 闭环、训练权重或生产部署；真实 MinIO/IAM/HTTPS 验收仍待完成。 未开启身份 API 时，/ready 只检查 I-01A 进程配置；开启后还检查迁移版本和 Redis 连通性，但仍不是完整系统就绪。开启上传也不会使 /ready 检查真实桶、IAM、签名请求或图像验证。 所有后端进程和前端构建均拒绝 production；AI_MODE 只接受 mock。 @@ -200,7 +200,7 @@ Redis URL 必须指向操作者已准备好的专用开发实例；上例是变 登录 `POST /api/v1/auth/login` 发送 JSON `{tenant_code,username,password}` 与 Origin；后续请求携带服务端 session cookie。`GET /api/v1/me` 获取 CSRF；创建用户/禁用/退出还需单个 X-CSRF-Token、Idempotency-Key、Origin 和 application/json。具体字段、响应和错误使用 [公共契约](contracts/public-api-v1.yaml)，不要将密码或 cookie 粘贴到日志/PR。 -本阶段不信任 X-Forwarded-For；反代后的 IP 限流按直接 peer 计数，不声称已识别真实终端 IP。身份及基础小请求暂限 64 KiB，模板创建为 2 MiB，以容纳 100 项完整 Unicode/转义正文；这不是未来上传/import 限额。/ready 只核对 `0001_initial` 与 Redis ping；模式结构仍必须用 CLI verify，存储、Worker、完整业务流程和真实模型均未纳入该门禁。生产/远端数据库仍被拒绝。 +本阶段不信任 X-Forwarded-For；反代后的 IP 限流按直接 peer 计数，不声称已识别真实终端 IP。身份及基础小请求暂限 64 KiB，模板创建为 2 MiB，以容纳 100 项完整 Unicode/转义正文；这不是未来上传/import 限额。/ready 只核对 `0002_report_object_version` 与 Redis ping；模式结构仍必须用 CLI verify，存储、Worker、完整业务流程和真实模型均未纳入该门禁。生产/远端数据库仍被拒绝。 ### 3.7 组织→模板→巡检草稿（I-02D） @@ -346,6 +346,31 @@ GET `/api/v1/images/{id}/download` 默认 analysis，`?variant=original` 需要 下载签发共用 120 次/分钟用户配额，不占 60 次/分钟写配额；Redis 故障返回 503，无普通元数据 GET 的本地降级。撤权后禁止再签发，已经签发的 URL 最长仍可用至 60 秒到期；不能宣称随 session 立即撤销。签发不检查存储网络和对象存在性，准确版本已丢失时对象 GET 可能失败，不能回退 latest 或修改 DB ready 状态。关闭上传开关并重启 API 可停止签发；保留历史审计与对象。证据与真实部署待验收项见 [I-02F4 验收](docs/08-delivery/21-i02f4-image-download.md)。 +### 3.15 CSV 报告执行（I-02I2） + +在已配置的专用 dev/test 库执行迁移至 head，再运行 verify。本批新增 `0002_report_object_version`，为报告增加 nullable object_version/size_bytes，保留原快照和初始迁移。未升级的库不会通过 `/ready`。 + +~~~powershell +.venv-business/Scripts/python.exe -m packages.persistence.cli upgrade +.venv-business/Scripts/python.exe -m packages.persistence.cli verify +$env:WORKER_DISPATCH_ENABLED = '1' +$env:WORKER_REPORT_EXPORT_ENABLED = '1' +~~~ + +三个独立终端分别运行以下命令，并在各终端设置上面的开关及现有数据库、Redis、S3 配置： + +~~~powershell +.venv-business/Scripts/python.exe -m apps.worker.dispatch publisher +.venv-business/Scripts/python.exe -m apps.worker.dispatch sweeper +.venv-business/Scripts/python.exe -m apps.worker.run +~~~ + +报告使用独立 `q.reports` 队列；仅开启报告开关时 Worker 只订阅该队列，同时启用图像/推理/规则开关时也订阅 q.general。publisher/sweeper 仅在报告开关开启时投递/补发 CSV；PDF 继续 queued/pending。报告开关默认关闭且拒绝 production。存储沿用独立 Worker secret，不回退 API 凭据；需 versioning 私有桶及授权租户/实验室 R 路径的 HEAD/GET/版本 GET/PUT 权限，不新增删除或列表权限。真实 IAM/TLS 尚待部署验收。 + +Worker 在无数据库连接的子进程内从冻结快照生成 CSV 并上传；最长 120 秒，10 秒心跳续租。最终事务重检输入/owner/token/generation 后登记精确对象版本、SHA、大小及 ready，expires_at 为提交时 DB now+24h。失败/kill 由持久任务重试与回收收敛；旧 worker 的上传可能留下未引用版本，不在执行路径删除对象。CSV 输出上限 64 MiB，格式不合规或超限终止为 SCHEMA_MISMATCH。 + +I-02I3 已接通 `GET /api/v1/reports/exports/{id}/download`。只有 ready、未过期 CSV 且对象 key/checksum/准确 VersionId/大小证据完整时签发；请求人必须是创建人或完整快照范围的 admin，并在当前事务重新具备每个实验室的 export 权限。签名固定 GET、同源、准确 VersionId 和 60 秒 TTL；不重新生成或读取业务数据，query 参数和跨租户/部分授权请求拒绝。PDF 渲染、完成通知、过期状态转换、精确对象清理、真实存储联调和 UI 仍待后续批次。停用时先停 publisher、排空报告 Worker，再关闭报告开关；保留 schema、对象版本及任务历史，勿在有数据的库执行 downgrade。详见 [报告执行验收](docs/08-delivery/29-i02-report-csv-execution.md) 与 [报告下载验收](docs/08-delivery/30-i02-report-download.md)。 + ## 4. 验证 根目录、业务环境执行：领域守卫也使用业务环境，不向 AI 环境引入业务授权或持久化依赖。 @@ -392,7 +417,7 @@ CI 必须先 --check，不能先生成来掩盖漂移。设计校验不是应用 I-01B 的实际结果与后续边界见 [验收记录](docs/08-delivery/08-i01b-acceptance.md)。I-01C 会话、RBAC、幂等与租户事务实现及验收范围见 [I-01C 验收记录](docs/08-delivery/09-i01c-acceptance.md)。 -截至已核验的历史状态，I-01A PR #5、I-01B PR #7、I-01C PR #8、I-02A–F2 PR #9 均已合并。当前在 `wsq/i-03a-durable-dispatch` 保留 A1/A2/F3/A3/F4 本地改动；本轮未提交、Push、创建 PR 或重新核验远程状态。数据库/安全变更仍需独立评审，不直接推送 main。 +历史 I-03A1/A2、I-02F3/F4、I-03A3 及截至第26批的工作已通过 PR #10 合并，2026-10-07 已 fetch 核验 origin/main=4e4f8bf。第27–30批整改/CSV报告使用 wsq/i-02-remediation-reports；各批验收中的未提交状态是实施当时的历史记录，当前提交/Push/CI状态以本次PR为准。合并前仍需独立审查；数据库/权限/模型变更由两位协作者共同确认，不自动合并。 I-02A 的领域守卫边界见 [领域命令守卫验收](docs/08-delivery/10-i02a-domain-guards.md)；身份 API、事务和历史故障验证见 [I-02B 验收](docs/08-delivery/11-i02b-identity-api.md)。当前 A1/A2/F3/A3/F4 未推送，不能将旧 PR 的 CI 成绩作为新分支已通过。 I-02C 的角色接口、作用域、版本与会话语义见 [角色 API 验收](docs/08-delivery/12-i02c-role-api.md)。角色请求的 expected_version 来自目标 User.version，laboratory_id 必须显式传 UUID/null；成功改权会撤销目标全部会话，包括操作者修改自身角色时的当前会话。三接口与 I-02B 共用开关和安全请求头，无新增配置。 I-02D 的接口、事务与本轮会话锁修复见 [基础 API 验收](docs/08-delivery/13-i02d-foundation-api.md)。实施计划的 I-0 表格同步记录本地验证状态，不把未提交工作记作远程 CI 或独立批准。 diff --git a/apps/api/app/jobs.py b/apps/api/app/jobs.py index dc33c09..43c8acc 100644 --- a/apps/api/app/jobs.py +++ b/apps/api/app/jobs.py @@ -1,4 +1,4 @@ -"""Strict job metadata and administrator replay adapters; validate_image branch only.""" +"""Strict image/report job metadata and administrator replay adapters.""" from uuid import UUID diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 683330a..2fe8c1a 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -10,6 +10,8 @@ from apps.api.app.inference_runs import mount_inference_runs from apps.api.app.item_queries import mount_item_queries from apps.api.app.jobs import mount_jobs +from apps.api.app.remediation import mount_remediation +from apps.api.app.reports import mount_reports from apps.api.app.review import mount_review from apps.api.app.rules import mount_rules from apps.api.app.uploads import mount_uploads @@ -63,6 +65,8 @@ async def lifespan(app): mount_jobs(app, identity, storage) mount_rules(app, identity) mount_review(app, identity) + mount_remediation(app, identity) + mount_reports(app, identity, storage) @app.get("/health") def health() -> dict[str, str | bool]: @@ -80,8 +84,9 @@ def ready() -> dict[str, str | bool]: "I-02B/C identity, I-02D foundations, I-02E item queries, " "I-03A3 image-job queries and optional replay, I-02G1 submit enqueue, " "optional I-02F1/F2 upload signing and validation acceptance only, " - "optional I-02F4 image download signing, and development rule " - "configuration lifecycle; " + "optional I-02F4 image download signing, I-02I2 CSV report " + "execution and I-02I3 report download signing, and development " + "rule configuration lifecycle; " "storage, inference execution and real-model readiness not implemented" ), } diff --git a/apps/api/app/remediation.py b/apps/api/app/remediation.py new file mode 100644 index 0000000..31eb329 --- /dev/null +++ b/apps/api/app/remediation.py @@ -0,0 +1,186 @@ +"""Strict remediation task and evidence API adapters.""" + +from datetime import datetime +from typing import Annotated, Literal +from uuid import UUID + +from fastapi import APIRouter, Request +from pydantic import BeforeValidator, Field + +from apps.api.app.foundations import Identifier +from apps.api.app.identity import Body, CanonicalJSONResponse, pagination +from packages.application.remediation import RemediationApplication +from packages.domain.security import ServiceError +from packages.persistence.security import SESSION_COOKIE + + +def _parse_datetime(value): + if isinstance(value, datetime): + parsed = value + elif isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ValueError("Invalid datetime") from error + else: + raise ValueError("Invalid datetime") + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("Datetime must include timezone") + return parsed + + +class RemediationCreate(Body): + expected_version: Annotated[int, Field(ge=1, le=2147483647)] + assignee_id: Identifier + due_at: Annotated[datetime, BeforeValidator(_parse_datetime)] + priority: Literal["low", "medium", "high", "critical"] + description: Annotated[str, Field(min_length=1, max_length=4000, pattern=r"\S")] + + +class EvidenceSubmit(Body): + expected_version: Annotated[int, Field(ge=1, le=2147483647)] + image_ids: Annotated[list[Identifier], Field(min_length=1, max_length=10)] + description: Annotated[str, Field(min_length=1, max_length=4000, pattern=r"\S")] + + +class Recheck(Body): + expected_version: Annotated[int, Field(ge=1, le=2147483647)] + evidence_id: Identifier + reason: Annotated[str, Field(min_length=1, max_length=2000, pattern=r"\S")] + + +class TaskAssignment(Body): + expected_version: Annotated[int, Field(ge=1, le=2147483647)] + assignee_id: Identifier + due_at: Annotated[datetime, BeforeValidator(_parse_datetime)] + reason: Annotated[str, Field(min_length=1, max_length=2000, pattern=r"\S")] + + +class Reason(Body): + expected_version: Annotated[int, Field(ge=1, le=2147483647)] + reason: Annotated[str, Field(min_length=1, max_length=2000, pattern=r"\S")] + + +def mount_remediation(app, identity): + app.state.remediation = RemediationApplication(identity) if identity is not None else None + router = APIRouter(prefix="/api/v1") + + def context(request, allowed=()): + if any( + k not in allowed or len(request.query_params.getlist(k)) != 1 + for k in request.query_params + ): + raise ServiceError("VALIDATION_ERROR", 422, "Unexpected or duplicate query parameter") + if app.state.remediation is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Remediation service not configured") + return app.state.remediation + + def read(request, kind, resource_id=None): + allowed = ( + () + if resource_id + else ( + "page", + "page_size", + "laboratory_id", + "status", + "overdue", + ) + ) + if kind == "evidence": + allowed = ("page", "page_size", "laboratory_id") + service = context(request, allowed) + page, size = pagination(request) + lab = request.query_params.get("laboratory_id") + if lab: + try: + lab = str(UUID(lab)) + except (TypeError, ValueError): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid laboratory_id") from None + status = request.query_params.get("status") + if status and status not in { + "pending_dispatch", + "in_progress", + "pending_recheck", + "rejected", + "cannot_remediate", + "closed", + }: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid remediation status") + overdue = request.query_params.get("overdue") + if overdue is not None and overdue not in {"true", "false"}: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid overdue filter") + return service.read( + kind, + request.cookies.get(SESSION_COOKIE), + request.state.request_id, + resource_id=str(resource_id) if resource_id else None, + laboratory_id=lab, + page=page, + page_size=size, + status=status, + overdue=None if overdue is None else overdue == "true", + ) + + def write(request, operation, resource_id, body): + status, response = context(request).write( + operation, + request.cookies.get(SESSION_COOKIE), + request.headers["x-csrf-token"], + request.headers["idempotency-key"], + str(resource_id), + body.model_dump(mode="json"), + request.state.request_id, + ) + return CanonicalJSONResponse(response, status_code=status) + + @router.post( + "/findings/{finding_id}/remediation-tasks", + status_code=201, + operation_id="dispatchRemediation", + ) + def dispatch(finding_id: UUID, body: RemediationCreate, request: Request): + return write(request, "dispatch", finding_id, body) + + @router.get("/remediation-tasks", operation_id="listRemediations") + def tasks(request: Request): + return read(request, "tasks") + + @router.get("/remediation-tasks/{task_id}", operation_id="getRemediation") + def task(task_id: UUID, request: Request): + return read(request, "task", task_id) + + @router.post("/remediation-tasks/{task_id}/accept", operation_id="acceptRemediation") + def accept(task_id: UUID, body: Reason, request: Request): + return write(request, "accept", task_id, body) + + @router.post( + "/remediation-tasks/{task_id}/submit-evidence", + operation_id="submitevidenceRemediation", + ) + def evidence(task_id: UUID, body: EvidenceSubmit, request: Request): + return write(request, "submit-evidence", task_id, body) + + @router.post("/remediation-tasks/{task_id}/recheck", operation_id="recheckRemediation") + def recheck(task_id: UUID, body: Recheck, request: Request): + return write(request, "recheck", task_id, body) + + @router.post("/remediation-tasks/{task_id}/reject", operation_id="rejectRemediation") + def reject(task_id: UUID, body: Recheck, request: Request): + return write(request, "reject", task_id, body) + + @router.post( + "/remediation-tasks/{task_id}/cannot-remediate", operation_id="cannotremediateRemediation" + ) + def cannot(task_id: UUID, body: Reason, request: Request): + return write(request, "cannot-remediate", task_id, body) + + @router.post("/remediation-tasks/{task_id}/reassign", operation_id="reassignRemediation") + def reassign(task_id: UUID, body: TaskAssignment, request: Request): + return write(request, "reassign", task_id, body) + + @router.get("/remediation-tasks/{task_id}/evidence", operation_id="listEvidence") + def evidence_list(task_id: UUID, request: Request): + return read(request, "evidence", task_id) + + app.include_router(router) diff --git a/apps/api/app/reports.py b/apps/api/app/reports.py new file mode 100644 index 0000000..d9ed3f5 --- /dev/null +++ b/apps/api/app/reports.py @@ -0,0 +1,88 @@ +"""Strict report snapshot creation and status query adapters.""" + +from typing import Annotated, Literal +from uuid import UUID + +from fastapi import APIRouter, Request +from pydantic import Field, field_validator + +from apps.api.app.foundations import Identifier +from apps.api.app.identity import Body, CanonicalJSONResponse +from packages.application.reports import ReportApplication +from packages.domain.security import ServiceError +from packages.domain.uploads import capture_time +from packages.persistence.security import SESSION_COOKIE + + +class ExportFilter(Body): + laboratory_ids: Annotated[list[Identifier], Field(min_length=1, max_length=100)] + from_: str = Field(alias="from") + to: str + severity: Annotated[list[Literal["low", "medium", "high", "critical"]], Field(max_length=4)] + finding_status: Annotated[ + list[ + Literal[ + "needs_review", "confirmed", "rejected", "cannot_determine", "dispatched", "closed" + ] + ], + Field(max_length=6), + ] + + @field_validator("from_", "to") + @classmethod + def date_time(cls, value): + capture_time(value) + return value + + +class ExportCreate(Body): + format: Literal["csv", "pdf"] + filters: ExportFilter + + +def mount_reports(app, identity, storage=None): + app.state.reports = ReportApplication(identity, storage) if identity is not None else None + router = APIRouter(prefix="/api/v1") + + def service(request): + if request.query_params: + raise ServiceError("VALIDATION_ERROR", 422, "Report endpoints take no query parameters") + if app.state.reports is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Report service not configured") + return app.state.reports + + def download_service(request): + if request.query_params: + raise ServiceError("VALIDATION_ERROR", 422, "Report endpoints take no query parameters") + report_service = service(request) + if report_service.storage is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Report downloads not configured") + return report_service + + @router.post("/reports/exports", status_code=202, operation_id="createExport") + def create(body: ExportCreate, request: Request): + status, response = service(request).create( + request.cookies.get(SESSION_COOKIE), + request.headers["x-csrf-token"], + request.headers["idempotency-key"], + body.model_dump(mode="json", by_alias=True), + request.state.request_id, + ) + return CanonicalJSONResponse(response, status_code=status) + + @router.get("/reports/exports/{export_id}", operation_id="getExport") + def get(export_id: UUID, request: Request): + return service(request).read( + request.cookies.get(SESSION_COOKIE), request.state.request_id, str(export_id) + ) + + @router.get("/reports/exports/{export_id}/download", operation_id="downloadExport") + def download(export_id: UUID, request: Request): + return CanonicalJSONResponse( + download_service(request).download( + request.cookies.get(SESSION_COOKIE), str(export_id), request.state.request_id + ), + headers={"Referrer-Policy": "no-referrer"}, + ) + + app.include_router(router) diff --git a/apps/worker/app/main.py b/apps/worker/app/main.py index 228e05c..1688278 100644 --- a/apps/worker/app/main.py +++ b/apps/worker/app/main.py @@ -35,16 +35,22 @@ def create_celery_app() -> Celery: os.getenv("WORKER_IMAGE_VALIDATION_ENABLED", "0") == "1" or os.getenv("WORKER_INFERENCE_ENABLED", "0") == "1" or os.getenv("WORKER_RULE_EVALUATION_ENABLED", "0") == "1" + or os.getenv("WORKER_REPORT_EXPORT_ENABLED", "0") == "1" ): from apps.worker.inference_pipeline import consume_inference from apps.worker.rule_evaluation import consume_rule image_enabled = os.getenv("WORKER_IMAGE_VALIDATION_ENABLED", "0") == "1" - if image_enabled: - from apps.worker.image_validation import consume_image + report_enabled = os.getenv("WORKER_REPORT_EXPORT_ENABLED", "0") == "1" + if report_enabled: + from apps.worker.report_export import consume_report + + if image_enabled or report_enabled: from packages.storage.s3 import S3Settings S3Settings.from_environment(os.environ["PUBLIC_ORIGIN"], worker=True) + if image_enabled: + from apps.worker.image_validation import consume_image @celery.task(name="labsafe.tasks.dispatch", ignore_result=True, shared=False) def dispatch(message): @@ -53,6 +59,10 @@ def dispatch(message): consume_inference(message) elif message.get("task_type") == "rule_evaluation": consume_rule(message) + elif message.get("task_type") == "report_export": + if not report_enabled: + raise RuntimeError("report_export consumer is disabled") + consume_report(message) else: if not image_enabled: raise RuntimeError("validate_image consumer is disabled") diff --git a/apps/worker/dispatch.py b/apps/worker/dispatch.py index e93d955..80d4d45 100644 --- a/apps/worker/dispatch.py +++ b/apps/worker/dispatch.py @@ -11,6 +11,7 @@ from packages.application.dispatch import DispatchPublisher, sweep_dispatch from packages.application.inference_execution import InferenceExecution from packages.application.job_execution import ImageExecution +from packages.application.report_execution import ReportExecution from packages.application.rule_execution import RuleExecution from packages.persistence.database import database_engine from packages.shared.environment import development_environment @@ -27,7 +28,7 @@ def send_dispatch(event_id, message): "labsafe.tasks.dispatch", args=[message], task_id=event_id, - queue="q.general", + queue="q.reports" if message.get("task_type") == "report_export" else "q.general", serializer="json", retry=False, headers={"outbox_event_id": event_id}, @@ -89,7 +90,8 @@ def main(): raise SystemExit("Set WORKER_DISPATCH_ENABLED=1 for development dispatch") logging.basicConfig(level=logging.INFO) engine = database_engine() - publisher = DispatchPublisher(engine, CeleryDispatchTransport()) + report_enabled = os.getenv("WORKER_REPORT_EXPORT_ENABLED", "0") == "1" + publisher = DispatchPublisher(engine, CeleryDispatchTransport(), report_enabled=report_enabled) try: while True: try: @@ -100,7 +102,8 @@ def main(): expired = ImageExecution(engine).recover_expired() expired += InferenceExecution(engine).recover_expired() expired += RuleExecution(engine).recover_expired() - recovered, resent = sweep_dispatch(engine) + expired += ReportExecution(engine).recover_expired() + recovered, resent = sweep_dispatch(engine, report_enabled=report_enabled) logging.info( "execution_recovered=%d dispatch_recovered=%d redispatched=%d", expired, diff --git a/apps/worker/report_export.py b/apps/worker/report_export.py new file mode 100644 index 0000000..5acccb0 --- /dev/null +++ b/apps/worker/report_export.py @@ -0,0 +1,109 @@ +"""CSV generation/upload child with a wall deadline and no database handles.""" + +import multiprocessing +import os +import time + +from packages.domain.job_execution import LeaseLost +from packages.domain.report_execution import ( + REPORT_ERRORS, + REPORT_SECONDS, + ReportArtifact, + ReportInvalid, + csv_bytes, + validate_artifact, +) +from packages.domain.security import ServiceError + + +def prepare_child(channel, settings, source): + from packages.storage.s3 import S3Storage + + storage = None + try: + payload = csv_bytes(source) + storage = S3Storage(settings) + channel.send(("result", storage.put_report(source, payload))) + except ReportInvalid: + channel.send(("error", "SCHEMA_MISMATCH")) + except ServiceError as error: + channel.send(("error", error.code if error.code in REPORT_ERRORS else "INTERNAL_ERROR")) + except Exception: + channel.send(("error", "INTERNAL_ERROR")) + finally: + channel.close() + if storage is not None: + storage.close() + + +class BoundedReportPrepare: + def __init__(self, settings): + self.settings = settings + + def __call__(self, source, cancelled): + if cancelled.is_set(): + raise LeaseLost() + context = multiprocessing.get_context("spawn") + receive, send = context.Pipe(duplex=False) + child = context.Process( + target=prepare_child, args=(send, self.settings, source), daemon=True + ) + started = time.monotonic() + try: + child.start() + send.close() + while True: + if cancelled.is_set(): + raise LeaseLost() + remaining = REPORT_SECONDS - (time.monotonic() - started) + if remaining <= 0: + raise ServiceError("STAGE_TIMEOUT", 504, "Report preparation timed out") + if receive.poll(min(0.1, remaining)): + try: + kind, value = receive.recv() + except EOFError: + raise ServiceError("INTERNAL_ERROR", 500, "Report child exited") from None + if cancelled.is_set(): + raise LeaseLost() + if time.monotonic() - started >= REPORT_SECONDS: + raise ServiceError("STAGE_TIMEOUT", 504, "Report preparation timed out") + if kind == "result" and isinstance(value, ReportArtifact): + validate_artifact(source, value) + return value + code = ( + value + if kind == "error" and isinstance(value, str) and value in REPORT_ERRORS + else "INTERNAL_ERROR" + ) + raise ServiceError( + code, + 503 if code == "DEPENDENCY_UNAVAILABLE" else 500, + "Report preparation failed", + ) + if not child.is_alive(): + raise ServiceError("INTERNAL_ERROR", 500, "Report child exited") + finally: + send.close() + receive.close() + if child.pid is not None: + if child.is_alive(): + child.kill() + child.join(timeout=2) + if child.is_alive(): + raise ServiceError("STAGE_TIMEOUT", 504, "Report child did not stop") + child.close() + + +def consume_report(message): + from packages.application.report_execution import ReportExecution + from packages.persistence.database import database_engine + from packages.persistence.dispatch import validate_dispatch + from packages.storage.s3 import S3Settings + + validate_dispatch(message) + settings = S3Settings.from_environment(os.environ["PUBLIC_ORIGIN"], worker=True) + engine = database_engine() + try: + return ReportExecution(engine).execute(message, BoundedReportPrepare(settings)) + finally: + engine.dispose() diff --git a/apps/worker/run.py b/apps/worker/run.py index 1119e5b..4743eec 100644 --- a/apps/worker/run.py +++ b/apps/worker/run.py @@ -19,13 +19,16 @@ def main() -> None: for name in ("WORKER_IMAGE_VALIDATION_ENABLED", "WORKER_INFERENCE_ENABLED") + ("WORKER_RULE_EVALUATION_ENABLED",) ) + queues = ["q.general"] if general else [] + if os.getenv("WORKER_REPORT_EXPORT_ENABLED", "0") == "1": + queues.append("q.reports") app.worker_main( [ "worker", "--loglevel=INFO", "--pool=solo", "--concurrency=1", - "--queues=q.general" if general else "--queues=celery", + "--queues=" + ",".join(queues or ["celery"]), ] ) diff --git a/contracts/data-model.json b/contracts/data-model.json index de6f9d8..101291b 100644 --- a/contracts/data-model.json +++ b/contracts/data-model.json @@ -5823,6 +5823,20 @@ "default_sql": null, "description": "", "generated_sql": null + }, + "object_version": { + "type": "VARCHAR(200)", + "nullable": true, + "default_sql": null, + "description": "", + "generated_sql": null + }, + "size_bytes": { + "type": "BIGINT UNSIGNED", + "nullable": true, + "default_sql": null, + "description": "", + "generated_sql": null } }, "primary_key": [ diff --git a/contracts/database-design.sql b/contracts/database-design.sql index 0f5c257..7a550c8 100644 --- a/contracts/database-design.sql +++ b/contracts/database-design.sql @@ -754,6 +754,8 @@ CREATE TABLE `report_exports` ( `checksum` CHAR(64) NULL, `expires_at` DATETIME(3) NULL, `last_error_code` VARCHAR(80) NULL, + `object_version` VARCHAR(200) NULL, + `size_bytes` BIGINT UNSIGNED NULL, PRIMARY KEY (`id`), UNIQUE KEY `uq_report_exports_0` (`tenant_id`, `id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/docs/02-architecture/05-durable-jobs.md b/docs/02-architecture/05-durable-jobs.md index 9cfb717..8f15f74 100644 --- a/docs/02-architecture/05-durable-jobs.md +++ b/docs/02-architecture/05-durable-jobs.md @@ -61,6 +61,8 @@ replayJob 仅 admin，expected_version 匹配且 state=dead_letter/failed。先 所有对象写入采用确定 key + checksum 后登记；重试发现同hash复用，异hash报错。孤儿对象清理必须以 DB 引用核验为准，不按目录粗暴删除。 +I-02I2 的 CSV report_export 使用 report→task 锁序、60 秒租约与独立 10 秒心跳；无 DB 句柄的生成/上传子进程最长 120 秒。提交核对冻结输入及聚合版本、owner/token/generation/attempt/租约有效期，原子登记 key/checksum/object_version/size_bytes、ready、DB now+24h 的 expires_at、系统审计和 task/attempt succeeded。可重试错误或过期回收恢复 report queued；第 4 次耗尽或永久错误置 failed，旧 attempts 保留。报告采用 dev/test opt-in q.reports，发布/补发按 DB format=csv 筛选，PDF 仍 pending。I-02I3 的 downloadExport 在当前租户/报告锁内重新核验创建人或同范围 admin、完整 export 权限、ready/24h 门禁和精确 key+VersionId，再在事务内只做本地 60 秒签名；不重生成、不读取对象。完成通知、过期处理仍待后批，详见 [I-02I2](../08-delivery/29-i02-report-csv-execution.md) 与 [I-02I3](../08-delivery/30-i02-report-download.md)。 + ## 6. 必测故障点 提交前kill→无业务/无事件；提交后发布前kill→sweeper补发；发布后mark前kill→重复事件由inbox防重；AI完成后DB提交前kill→可重复计算但仅当前token提交；DB提交后ack前kill→重复消息无新增结果；lease过期后旧结果→拒绝；Redis清空→30秒扫描+发布恢复；超过4次→dead_letter；手工重放→新generation且完整历史。 diff --git a/docs/02-architecture/06-data-persistence.md b/docs/02-architecture/06-data-persistence.md index a385bc2..765c9e6 100644 --- a/docs/02-architecture/06-data-persistence.md +++ b/docs/02-architecture/06-data-persistence.md @@ -4,6 +4,8 @@ 完整列/空值/默认值/索引/唯一键/外键见 [数据字典](../../contracts/data-model.json)，建表顺序与约束见 [设计 DDL](../../contracts/database-design.sql)。生成源 tools/design/data_model.py。目标是 MySQL 8.0.16+、InnoDB、utf8mb4_bin、UTC；不是已经运行过的生产迁移。 +I-02I2 新增前向迁移 `0002_report_object_version`：report_exports 增加 nullable object_version VARCHAR(200) 和 size_bytes BIGINT UNSIGNED，既有 key/checksum 与精确版本一起由围栏事务登记。初始 0001 迁移及快照保持冻结；结构检查和 API readiness 要求新 head。对已有报告行的升级保留原字段，新增字段为空，不为历史未登记对象猜测版本。 + DDL 先建表再加 FK 以解决 current pointer 环依赖。迁移使用 Alembic，先在空数据库执行并 introspection 对比字典，再测试同租户/跨租户数据；真实数据库未执行前只能报告静态验证。所有连接设置 session time_zone='+00:00'、严格 SQL mode；应用每次更新显式维护 updated_at/version。 ## 2. 核心约束 diff --git a/docs/03-interfaces/01-api-contract.md b/docs/03-interfaces/01-api-contract.md index 8e9e53d..abb1554 100644 --- a/docs/03-interfaces/01-api-contract.md +++ b/docs/03-interfaces/01-api-contract.md @@ -111,10 +111,10 @@ CSV列固定：inspection_id,item_id,laboratory_id,location_id,run_id,fact_revis ### 5.7 图像验证结果与受控下载（I-02F3/F4） - F3 的 opt-in general Worker 只消费 validate_image；准确 GET、实算 SHA、真实 Pillow 解码、一次 EXIF 旋转、去元数据 RGB PNG 后登记 O/A 和 ready。内容拒绝登记 image/upload=rejected、task=succeeded；技术故障保留 validating、自动重试/技术终态。ImageValidated 只在成功时写 Outbox，下游发布/inbox 未接通。见 [F3 验收](../08-delivery/19-i02f3-image-validation.md)。 -- F4 接通 GET /api/v1/images/{id}/download（downloadImage），默认 variant=analysis，original 额外 Admin+原图读取审计。两者均按图像所属实验室 READ，只有 ready 且所选 O/A key/SHA/版本完整可签发，非 ready 返回 409。只允许单个 variant 参数，拒绝客户端对象 key/version/tenant 或额外 query。 +- F4 接通 GET /api/v1/images/{id}/download（downloadImage），默认 variant=analysis，original 额外 Admin+原图读取审计。两者均按图像所属实验室 READ，只有 ready 且所选 O/A key/SHA/版本完整可签发，非 ready 返回 409。只允许单个 variant 参数，拒绝客户端对象 key/version/tenant 或额外 query。I-02I3 接通 GET /api/v1/reports/exports/{id}/download：ready、24h 内的 CSV 才能按创建人或同范围 admin 签发，且完整快照范围重新具备 export 权限；签名固定准确 key+VersionId、60 秒，不重新生成。 - DownloadGrant 仅 `{url,expires_at}`；同源 HTTPS SigV4 GET、固定数据库 VersionId、60 秒。签名原样使用，不跳到最新对象。API 返回 no-store/no-referrer，不重定向；原图审计与签发事务一起提交，审计失败不返回 grant。GET 不要求写命令头；每次原图签发独立审计，不做幂等缓存。 - 当前身份/角色/session 和图片记录在事务内锁定，SDK 签名为本地操作，无 HEAD/GET 或凭据刷新。下载不改变 image/upload/task/event。签发不证明对象仍存在或真实 IAM/TLS 可用；对象 GET 失败不能据此更换 VersionId 或伪造成功。 -- 共用 120 次/分钟用户限流，Redis 故障不签发；不占写配额，不使用普通元数据读的降级。撤权后禁止新签发，既有 URL 到期前最长 60 秒仍可能有效。沿用 API_UPLOADS_ENABLED 和 API S3 secret；无新增迁移/配置。见 [F4 验收](../08-delivery/21-i02f4-image-download.md)。crop/报告下载与真实部署验收未完成。 +- 共用 120 次/分钟用户限流，Redis 故障不签发；不占写配额，不使用普通元数据读的降级。撤权后禁止新签发，既有 URL 到期前最长 60 秒仍可能有效。沿用 API_UPLOADS_ENABLED 和 API S3 secret；无新增迁移/配置。见 [F4 验收](../08-delivery/21-i02f4-image-download.md)。crop 与真实部署验收仍未完成；报告下载见 [I-02I3 验收](../08-delivery/30-i02-report-download.md)。 ### 5.8 提交巡检项与推理入队（I-02G1） diff --git a/docs/08-delivery/01-implementation-plan.md b/docs/08-delivery/01-implementation-plan.md index 984d01e..e24d03f 100644 --- a/docs/08-delivery/01-implementation-plan.md +++ b/docs/08-delivery/01-implementation-plan.md @@ -1,40 +1,51 @@ # 08.1 实施计划与首批任务 -状态（2026-10-05）：I-01A/B/C 已合并；I-02A/B/C/D/E/F1/F2 已由 PR #9 合并。I-03A1 调度、I-03A2 上传验证/推理/规则执行基础、I-02F3 图像处理、I-03A3 图像任务查询/管理员重放、I-02F4 受控图片下载、规则管理和事实/人工复核代码已在本地完成；真实存储部署验收、remediation、report/export 和其余业务命令仍待关闭，完成情况见表及本批验收。整个 I-02/I-03 尚未完成；业务闭环、真实模型和生产发布不得以局部成绩代替验收。阶段标准见[阶段门禁](05-stage-gates.md)，当前入口见[开发指南](../../DEVELOPMENT.md)。 +状态（2026-10-07）：I-01A/B/C 已合并；I-02A/B/C/D/E/F1/F2 已由 PR #9 合并。此前本地批次已接调度、图像/推理/规则执行、图片下载、规则管理、人工复核、整改和报告快照 API；本批 I-02I2 接通 CSV 报告发布/执行/恢复及精确对象版本登记，I-02I3 接通 ready CSV 的受控下载签发。PDF、完成通知、过期处理、对象清理、真实存储部署和其余业务命令仍待关闭。整个 I-02/I-03 尚未完成；业务闭环、真实模型和生产发布不得以局部成绩代替验收。阶段标准见[阶段门禁](05-stage-gates.md)，当前入口见[开发指南](../../DEVELOPMENT.md)。 ## 1. 从“只有文档”开始 不要求用户先提供代码、训练后权重或运行报告。先创建受控开发环境和最小业务链路；真实模型/数据准备可以与业务代码并行，但任务仍按各自依赖验收。 -| 编号 | 任务与输入 | 必须交付 | 当前进度与证据 | 完成检查 / 失败处理 | -| ---------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | -| I-01A | 编码基线；APP_ENV=dev；Python 3.11 独立业务/AI 环境 | API、Worker、AI、Web 骨架与进程入口、环境样例、依赖约束、CI | 已合并 PR #5；见[骨架验收](07-i01a-acceptance.md)，不是完整业务服务 | 干净环境 import/HTTP smoke/build；缺框架必须失败，不安装真实模型也可启动同协议 fixture | -| I-01B | 设计 DDL；专用可丢弃 MySQL | Alembic 初始迁移、bootstrap CLI、结构核对 | 已合并 PR #7；43 表、115 外键、迁移循环及约束验证，见[数据库验收](08-i01b-acceptance.md) | 空库 upgrade/verify；失败仅处理专用测试库，不触碰已有库 | -| I-01C | 数据约束和安全规范 | 基础仓储/租户事务、session/RBAC、幂等、安全原语测试 | 已合并 PR #8；见[安全原语验收](09-i01c-acceptance.md)；HTTP/并发端到端证据在 I-02 分批补齐 | DB-01/02、API-02/03、SEC-01/02 按实际覆盖记证；失败回滚，无旁路权限 | -| I-02 | 状态机与公共契约 | 巡检、上传、事实、人工复核、整改等命令/查询及认证管理 API | **进行中**；A/B/C/D/E/F1/F2 已合并 PR #9；含本地 A3 的 3 个 validate_image 接口、F4 下载、G1 submit、规则配置和事实/复核 API；本批业务/领域/协议 1477 项、完整持久化 MySQL 386 项通过；整改、report/export 和完整 UI 仍待完成 | 每个 guard 正/反例；锁定完整写集；相同请求重发无重复副作用；不能以身份/草稿/查询/上传接口验收整个 I-02 | -| I-02A | 领域状态机与守卫 | 13 个纯领域命令守卫、权限/版本/状态/证据测试 | 本地完成；542 个领域用例，见[领域守卫验收](10-i02a-domain-guards.md) | 后续应用事务加载可信完整聚合；纯守卫放行不等于数据库已提交 | -| I-02B | 认证和用户契约、安全原语 | 登录/会话/退出、用户列表/查询/创建/禁用，共 7 个 API；事务、限流、审计/幂等与故障验证 | 本地完成；597 项单元/边界、67 项持久化套件（有重叠），见[身份 API 验收](11-i02b-identity-api.md) | 同 key/版本并发、撤权后重放拒绝、最后管理员保护、失败原子回滚；仅 dev/test | -| I-02C | 角色契约与 I-02B 事务链路 | 角色列表/授予/撤销 3 个 API；精确 scope、version/epoch、撤会话、最后管理员及幂等保护 | 本地完成；612 项累计单元/边界、93 项持久化套件，其中新增 26 项角色集成测试；见[角色 API 验收](12-i02c-role-api.md) | 六角色 scope、跨租户/实验室拒绝、同 key/版本竞争、双管理员撤权、失败回滚与权限投影容量守卫 | -| I-02D | 组织、模板与巡检创建契约 | 学院/实验室/位置、模板创建/发布/克隆、巡检草稿 16 个 API；原子子项与作用域隔离 | 本地完成；691 项累计单元/边界、127 项持久化套件；本批新增 79 项单元/边界与 34 项数据库用例，见[基础 API 验收](13-i02d-foundation-api.md) | 父类型/active、发布和 family revision、≤100 项、同 key/多会话竞争、权限计数与全事务回滚；不包括上传/完成/取消 | -| I-02E | 巡检项查询契约与同源领域守卫 | 列表/详情 2 个 GET；白名单投影、授权后分页、动作探测基础与能力门禁 | 本地完成；961 项累计单元/边界、154 项持久化套件；新增 270 项单元/边界与 27 项数据库用例，见[巡检项查询验收](14-i02e-item-query-api.md) | 租户/实验室隔离、RR 快照、无业务副作用、未知上下文不放行；未接写命令，allowed_actions 暂为空 | -| I-02F1 | 上传授权契约、采集权限与同源 S3 规范 | createUpload、双额度守卫、原子 grant/审计/幂等；真实 SDK 本地签名及固定版本/SHA 适配基础 | 本地完成；1227 项累计单元/边界、178 项持久化套件（157 项真实 MySQL）；新增 266 项单元/边界与 24 项数据库用例，见[上传授权验收](15-i02f1-upload-grant-api.md)；默认关闭，仅 dev/test | 同用户并发不超额、重放不续签、撤权拒绝、失败全回滚；未实现 complete/解码/ready 或真实存储联调 | -| I-02F2 | F1 与持久任务/图像契约 | completeUpload/getImage、固定 staging version、原子 image+validate_image 任务/dispatch | 本地完成；1283 项主套件、213 项完整持久化回归（192 项真实 MySQL + 21 项重叠单元），本批新增 35 项数据库用例，见[上传完成验收](16-i02f2-upload-completion-api.md)；默认关闭 | 重复 complete 不重复建图/任务，外部 HEAD 在事务外、提交前重验；对接 general 消费、图像变换和真实存储验收后方可 ready | -| I-03 | 持久任务设计 | Outbox/inbox/publisher/sweeper、Worker、fixture-v1 AI 服务执行闭环 | **进行中**；A1/A2/F3/A3 已接 validate_image 调度/执行/图像消费者/人工重放；仍缺领域事件发布、其他任务领域适配及其 replay、inbox/AI 闭环 | JOB-01/02/03/04 故障注入；断 Redis 后从 DB 恢复；不能用同步路由或 echo 任务替代 | -| I-03A1 | F2 的 validate_image TaskDispatch | 发布租约、事务外有界发布、过期发布回收、从 DB 补发至 q.general | 本地完成；1290 项业务/领域/安全、239 项完整持久化（218 MySQL + 21 单元）通过；新增 28 + 26 项，见[持久调度验收](17-i03a1-durable-dispatch.md)；默认关闭，未提交/Push/PR | SKIP LOCKED 并发、过期 owner 拒绝、插入失败回滚、真实 Redis 丢消息恢复；不标图像 ready | -| I-03A2 | A1 与领域任务入口 | 执行 lease/attempt/heartbeat/fencing、失败收敛、任务回收与 replay 基础 | **部分实现**；validate_image、inference_pipeline 与 rule_evaluation 已接专属执行基础；推理见[推理执行验收](23-i03a2-inference-execution.md)，规则见[规则评估验收](24-i03a2-rule-evaluation.md)；report 仍待接 | 聚合先于 task 加锁，旧租约/取消拒绝、4 次耗尽；report 任务与完整 JOB-04 仍待实现 | -| I-02F3 | F2、A1 与 A2 图像执行基础 | general 验证消费、固定版本/SHA、图像变换、O/A 与完整 ready/rejected 事务 | **本地代码完成**；1360 项主套件 + 4 协议 + 21 单元、318 项完整持久化（297 MySQL + 21 重叠单元）通过；新增 49 + 37 项，见[图像验证验收](19-i02f3-image-validation.md)；默认关闭，未提交/Push/PR；真实存储验收待 | 真实 Pillow、准确对象版本、同 SHA 复用、原子状态/审计/事件与旧 attempt 拒绝已测；UP-03/SEC-03 部署验收仍待 | -| I-03A3 | A2/F3 与 Job 公共契约 | validate_image 任务详情、技术终态列表、管理员固定输入重放；generation/fence/dispatch/审计/幂等同事务 | **本地代码完成**；1429 项主套件/协议/单元、357 项完整持久化（336 MySQL + 21 重叠单元）通过；新增 44 + 39 项，见[图像任务重放验收](20-i03a3-image-job-replay.md)；未提交/Push/PR | HEAD 在事务外；重验 Admin/归属/输入版本；旧 generation/lease 拒绝；历史保留；其他类型未开放 | -| I-02F4 | F3 ready 图像与 downloadImage 契约 | 分析图 READ、原图 Admin+审计；规范 O/A 固定版本同源 60 秒 GET 签名 | **本地代码完成**；1469 项主套件/协议/单元、383 项完整持久化（362 MySQL + 21 重叠单元）通过；新增 40 + 26 项，见[受控图片下载验收](21-i02f4-image-download.md)；未提交/Push/PR | 独立 HMAC 核签；锁内当前鉴权；非 ready/坏引用不签名；审计失败不返回 grant；Redis 故障不降级；真实 IAM/TLS 待验收 | -| I-02G1 | submitInspectionItem 与推理任务入队 | 可信 item/image/activation 锁定；旧 run/finding supersede；新 run、run_images、inference_pipeline task、TaskDispatch、审计、幂等同事务 | **本地代码完成**；新增 API/持久化验收与 task-message-v1 校验；既有 383 项真实 MySQL 回归通过，本批隔离 MySQL 专测 1 项通过，见[提交入队验收](22-i02g1-submit-inference-enqueue.md) | activation 只取已发布 model/dictionary；图片清单仅客户端选择，模型/规则/设备/日期服务端冻结；规则评估由后续专属任务接续；真实生产闭环仍未完成 | +| 编号 | 任务与输入 | 必须交付 | 当前进度与证据 | 完成检查 / 失败处理 | +| ---------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | +| I-01A | 编码基线；APP_ENV=dev；Python 3.11 独立业务/AI 环境 | API、Worker、AI、Web 骨架与进程入口、环境样例、依赖约束、CI | 已合并 PR #5；见[骨架验收](07-i01a-acceptance.md)，不是完整业务服务 | 干净环境 import/HTTP smoke/build；缺框架必须失败，不安装真实模型也可启动同协议 fixture | +| I-01B | 设计 DDL；专用可丢弃 MySQL | Alembic 初始迁移、bootstrap CLI、结构核对 | 已合并 PR #7；43 表、115 外键、迁移循环及约束验证，见[数据库验收](08-i01b-acceptance.md) | 空库 upgrade/verify；失败仅处理专用测试库，不触碰已有库 | +| I-01C | 数据约束和安全规范 | 基础仓储/租户事务、session/RBAC、幂等、安全原语测试 | 已合并 PR #8；见[安全原语验收](09-i01c-acceptance.md)；HTTP/并发端到端证据在 I-02 分批补齐 | DB-01/02、API-02/03、SEC-01/02 按实际覆盖记证；失败回滚，无旁路权限 | +| I-02 | 状态机与公共契约 | 巡检、上传、事实、人工复核、整改及报告受理等命令/查询与认证管理 API | **进行中**；A/B/C/D/E/F1/F2 已合并 PR #9；本地新增整改及 report 快照受理；本轮业务/领域/协议 1500 项通过，见 [I-02H](27-i02-remediation.md) 和 [I-02I1](28-i02-report-export.md)；导出执行、下载及完整 UI 待完成 | guard 正/反例、完整写集、同 key 无重复副作用；局部 API 不代表整个 I-02 完成 | +| I-02A | 领域状态机与守卫 | 13 个纯领域命令守卫、权限/版本/状态/证据测试 | 本地完成；542 个领域用例，见[领域守卫验收](10-i02a-domain-guards.md) | 后续应用事务加载可信完整聚合；纯守卫放行不等于数据库已提交 | +| I-02B | 认证和用户契约、安全原语 | 登录/会话/退出、用户列表/查询/创建/禁用，共 7 个 API；事务、限流、审计/幂等与故障验证 | 本地完成；597 项单元/边界、67 项持久化套件（有重叠），见[身份 API 验收](11-i02b-identity-api.md) | 同 key/版本并发、撤权后重放拒绝、最后管理员保护、失败原子回滚；仅 dev/test | +| I-02C | 角色契约与 I-02B 事务链路 | 角色列表/授予/撤销 3 个 API；精确 scope、version/epoch、撤会话、最后管理员及幂等保护 | 本地完成；612 项累计单元/边界、93 项持久化套件，其中新增 26 项角色集成测试；见[角色 API 验收](12-i02c-role-api.md) | 六角色 scope、跨租户/实验室拒绝、同 key/版本竞争、双管理员撤权、失败回滚与权限投影容量守卫 | +| I-02D | 组织、模板与巡检创建契约 | 学院/实验室/位置、模板创建/发布/克隆、巡检草稿 16 个 API；原子子项与作用域隔离 | 本地完成；691 项累计单元/边界、127 项持久化套件；本批新增 79 项单元/边界与 34 项数据库用例，见[基础 API 验收](13-i02d-foundation-api.md) | 父类型/active、发布和 family revision、≤100 项、同 key/多会话竞争、权限计数与全事务回滚；不包括上传/完成/取消 | +| I-02E | 巡检项查询契约与同源领域守卫 | 列表/详情 2 个 GET；白名单投影、授权后分页、动作探测基础与能力门禁 | 本地完成；961 项累计单元/边界、154 项持久化套件；新增 270 项单元/边界与 27 项数据库用例，见[巡检项查询验收](14-i02e-item-query-api.md) | 租户/实验室隔离、RR 快照、无业务副作用、未知上下文不放行；未接写命令，allowed_actions 暂为空 | +| I-02F1 | 上传授权契约、采集权限与同源 S3 规范 | createUpload、双额度守卫、原子 grant/审计/幂等；真实 SDK 本地签名及固定版本/SHA 适配基础 | 本地完成；1227 项累计单元/边界、178 项持久化套件（157 项真实 MySQL）；新增 266 项单元/边界与 24 项数据库用例，见[上传授权验收](15-i02f1-upload-grant-api.md)；默认关闭，仅 dev/test | 同用户并发不超额、重放不续签、撤权拒绝、失败全回滚；未实现 complete/解码/ready 或真实存储联调 | +| I-02F2 | F1 与持久任务/图像契约 | completeUpload/getImage、固定 staging version、原子 image+validate_image 任务/dispatch | 本地完成；1283 项主套件、213 项完整持久化回归（192 项真实 MySQL + 21 项重叠单元），本批新增 35 项数据库用例，见[上传完成验收](16-i02f2-upload-completion-api.md)；默认关闭 | 重复 complete 不重复建图/任务，外部 HEAD 在事务外、提交前重验；对接 general 消费、图像变换和真实存储验收后方可 ready | +| I-03 | 持久任务设计 | Outbox/inbox/publisher/sweeper、Worker、fixture-v1 AI 服务执行闭环 | **进行中**；A1/A2/F3/A3 已接 validate_image 调度/执行/图像消费者/人工重放；仍缺领域事件发布、其他任务领域适配及其 replay、inbox/AI 闭环 | JOB-01/02/03/04 故障注入；断 Redis 后从 DB 恢复；不能用同步路由或 echo 任务替代 | +| I-03A1 | F2 的 validate_image TaskDispatch | 发布租约、事务外有界发布、过期发布回收、从 DB 补发至 q.general | 本地完成；1290 项业务/领域/安全、239 项完整持久化（218 MySQL + 21 单元）通过；新增 28 + 26 项，见[持久调度验收](17-i03a1-durable-dispatch.md)；默认关闭，未提交/Push/PR | SKIP LOCKED 并发、过期 owner 拒绝、插入失败回滚、真实 Redis 丢消息恢复；不标图像 ready | +| I-03A2 | A1 与领域任务入口 | lease/attempt/heartbeat/fencing、失败收敛、回收与 replay | **部分实现**；validate_image、inference_pipeline、rule_evaluation 已接执行基础；report_export 本批仅登记任务及 failed replay，见 [报告受理验收](28-i02-report-export.md) | report 专属执行与回收、发布白名单和完整 JOB-04 尚未关闭 | +| I-02F3 | F2、A1 与 A2 图像执行基础 | general 验证消费、固定版本/SHA、图像变换、O/A 与完整 ready/rejected 事务 | **本地代码完成**；1360 项主套件 + 4 协议 + 21 单元、318 项完整持久化（297 MySQL + 21 重叠单元）通过；新增 49 + 37 项，见[图像验证验收](19-i02f3-image-validation.md)；默认关闭，未提交/Push/PR；真实存储验收待 | 真实 Pillow、准确对象版本、同 SHA 复用、原子状态/审计/事件与旧 attempt 拒绝已测；UP-03/SEC-03 部署验收仍待 | +| I-03A3 | A2/F3 与 Job 公共契约 | validate_image 任务详情、技术终态列表、管理员固定输入重放；generation/fence/dispatch/审计/幂等同事务 | **本地代码完成**；1429 项主套件/协议/单元、357 项完整持久化（336 MySQL + 21 重叠单元）通过；新增 44 + 39 项，见[图像任务重放验收](20-i03a3-image-job-replay.md)；未提交/Push/PR | HEAD 在事务外；重验 Admin/归属/输入版本；旧 generation/lease 拒绝；历史保留；其他类型未开放 | +| I-02F4 | F3 ready 图像与 downloadImage 契约 | 分析图 READ、原图 Admin+审计；规范 O/A 固定版本同源 60 秒 GET 签名 | **本地代码完成**；1469 项主套件/协议/单元、383 项完整持久化（362 MySQL + 21 重叠单元）通过；新增 40 + 26 项，见[受控图片下载验收](21-i02f4-image-download.md)；未提交/Push/PR | 独立 HMAC 核签；锁内当前鉴权；非 ready/坏引用不签名；审计失败不返回 grant；Redis 故障不降级；真实 IAM/TLS 待验收 | +| I-02G1 | submitInspectionItem 与推理任务入队 | 可信 item/image/activation 锁定；旧 run/finding supersede；新 run、run_images、inference_pipeline task、TaskDispatch、审计、幂等同事务 | **本地代码完成**；新增 API/持久化验收与 task-message-v1 校验；既有 383 项真实 MySQL 回归通过，本批隔离 MySQL 专测 1 项通过，见[提交入队验收](22-i02g1-submit-inference-enqueue.md) | activation 只取已发布 model/dictionary；图片清单仅客户端选择，模型/规则/设备/日期服务端冻结；规则评估由后续专属任务接续；真实生产闭环仍未完成 | +| I-02H | 整改任务与证据闭环 | confirmed finding 派发、受派人认领、ready 图片证据、独立复查/驳回、无法整改与转派；任务/ finding 双状态、审计、review_actions、outbox、通知、幂等同事务 | **本地代码完成**；严格 API、领域守卫、事务持久化和专项 MySQL 2 项通过，见[整改闭环验收](27-i02-remediation.md)；未提交/Push/PR | recheck/reject 必须排除受派人和证据提交人；图片必须属于 task 且 ready；过期/版本冲突和任一写失败必须整笔回滚；事件写入 outbox 不等于已发布 | +| I-02I1 | report/export 快照与异步受理 | 权限/容量守卫、冻结快照、report_export task、TaskDispatch/ReportRequested、状态查询和 failed 重放 | **本地实现**；新增 17 项单元/边界、21 项 MySQL 专项通过；受影响 MySQL 回归共 86 项通过，见 [报告受理验收](28-i02-report-export.md)；未提交/Push/PR | 31 天/100 实验室/10000 findings；完整范围授权；事务回滚；消费者/CSV/PDF/对象版本/下载/完成通知属于下一批 | | I-ML-01 | ML-BASE-02 固定提交/配方；不依赖完整 API 或项目权重 | D-FINE 四类开发 checkpoint、映射/安全加载/qmax、ONNX、训练/CPU/CUDA lock 与 smoke | 未完成；当前仅有模型设计与合成 fixture，不是 D-FINE-N 实际执行 | 分类头未训必须 development；RGB/W/H/300 query/设备核对；不兼容形成 ADR | | I-ML-02 | 数据来源授权与标注规范 | 授权台账、100 张标注试行、完整数据/split/审核 | 未完成；无本阶段真实数据验收证据 | 类别/scene 不泄漏；不足标待补充，无授权不抓取替代 | | I-ML-03/04 | I-ML-01/02 与验收 policy | 四类训练/校准、训练后 50 图 PyTorch/ORT CPU/CUDA 一致性与资源报告 | 未开始验收；依赖真实数据、制品和固定环境 | AI-05 至 AI-09、MODEL-02；检测 GPU/OCR CPU；不以初始化模型或 mock 代替 | -| I-04 | 稳定 API 和 fixture 服务 | 前端采集/复核/整改/admin/report，开发环境标识 | 未完成；只有开发 Web 骨架，尚无完整登录、角色管理或业务 UI 联调 | UI-01、EXPORT-01、FLOW 闭环；旧轮询不能覆盖新 run | -| I-05 | 已实现各服务 | Compose、secret 挂载、探针、日志/指标、备份恢复 | 未完成；开发入口/局部探针不等于完整部署 | 隔离栈闭环、恢复验证；按服务/数据定位，不改证据掩盖失败 | -| R-01 | 机构批准 policy/规则、固定真实模型与 test | 真实业务评测、恢复/断公网演练、双人发布记录 | 未进入；production 仍禁止 | EVAL/OPS/Security 全部通过；缺批准或指标不足禁止发布 | +| I-04 | 稳定 API 和 fixture 服务 | 前端采集/复核/整改/admin/report，开发环境标识 | 未完成；只有开发 Web 骨架，尚无完整登录、角色管理或业务 UI 联调 | UI-01、EXPORT-01、FLOW 闭环；旧轮询不能覆盖新 run | +| I-05 | 已实现各服务 | Compose、secret 挂载、探针、日志/指标、备份恢复 | 未完成；开发入口/局部探针不等于完整部署 | 隔离栈闭环、恢复验证；按服务/数据定位，不改证据掩盖失败 | +| R-01 | 机构批准 policy/规则、固定真实模型与 test | 真实业务评测、恢复/断公网演练、双人发布记录 | 未进入；production 仍禁止 | EVAL/OPS/Security 全部通过；缺批准或指标不足禁止发布 | 状态口径：已合并表示对应历史 PR 已合并；本地完成不代表已提交、远程 CI 通过或独立审查批准。I-02A/B/C/D/E/F1/F2 为 I-02 的子批次，表中各子批的本地成绩为历史验收记录，现均已随 #9 合并。统计中的套件重叠范围见各批验收记录，不累加成唯一测试数。F2 接受上传并登记元数据，A1/A2 接调度与执行，F3 已本地接通验证消费和 ready/rejected 事务，A3 补图像技术失败后的人工重放，F4 补 ready 图像受控下载签发，G1 接通提交入口但不声称推理已执行；真实存储链路、推理执行和结果闭环仍待验收。巡检项 allowed_actions 仍为空，直到查询端具备可信的默认图片选择上下文。 +### 2026-10-06 续批 I-02I2 + +上表 I-02I1 / I-03A2 的历史报告边界由本批接续：CSV 已有 report 专属执行租约、attempt、独立心跳、120 秒有界生成/上传、围栏提交、重试/回收、管理员 replay 后执行及 CSV 发布门禁。迁移 head 更新为 `0002_report_object_version`，初始迁移不变。当前业务/领域/安全/协议 1547 项、完整持久化 446 项（425 MySQL + 21 单元）全部通过；范围及证据见 [I-02I2 验收](29-i02-report-csv-execution.md)。 + +| 编号 | 本批交付 | 状态 | 后续边界 | +|---|---|---|---| +| I-02I2 | CSV 公式防注入、快照生成、精确对象版本/大小/SHA 登记、ready/24h expires_at、系统审计、q.reports 发布与恢复 | 本地实现；默认关闭，仅 dev/test，未提交/Push/PR | PDF、完成通知、expired 状态处理、对象清理、真实存储验收与 UI 尚未实现；不关闭 EXPORT-01/JOB-04 | +| I-02I3 | 创建人/同范围 admin 的完整快照范围重检、ready/expiry 门禁、精确 key+VersionId 的 60 秒 CSV 下载签名 | 本地实现；默认关闭，仅 dev/test，未提交/Push/PR；见 [报告下载验收](30-i02-report-download.md) | PDF、完成通知、expired 状态处理、对象清理、真实存储验收与 UI 尚未实现；不关闭 EXPORT-01/JOB-04 | + ## 2. 数据库验证的精确任务边界 目标MySQL8.0.16+、InnoDB、UTC、utf8mb4_bin。先DDL空库检查，再迁移实现和数据不变量测试；两者使用明确标注可丢弃的独立测试schema，凭据来自测试secret。结果记录数据库版本、迁移head、列/default/index/FK差异、每个非法插入的预期错误、应用事务拒绝路径及日志。 @@ -55,4 +66,4 @@ IRR六项设计前置按[关闭清单](06-implementation-readiness.md)执行。 ## 5. 协作和Push -设计基线与 I-01A/B/C 已通过各自 PR 合并；I-02A/B/C/D/E/F1/F2 已由 PR #9 合并。当前 I-03A1/A2、I-02F3、I-03A3 和 I-02F4 位于特性分支 `wsq/i-03a-durable-dispatch`；工作区保留各批改动，尚未提交/Push/PR。本轮没有重新 fetch 或核验远程合并状态。不直接推送 main，不自动合并或替代独立审查。各批验收中的未提交状态是当时的历史记录；当前交付状态以本节及 GitHub PR 为准。远程 CI 和数据库/权限变更的协作者审查仍为合并前置。 +历史 I-03A1/A2、I-02F3/F4、I-03A3 及截至第26批的工作已通过 PR #10 合并，2026-10-07 已 fetch 核验 origin/main=4e4f8bf。第27–30批整改/CSV报告使用 wsq/i-02-remediation-reports；各批验收中的未提交状态是实施当时的历史记录，当前提交/Push/CI状态以本次PR为准。合并前仍需独立审查；数据库/权限/模型变更由两位协作者共同确认，不自动合并。 diff --git a/docs/08-delivery/27-i02-remediation.md b/docs/08-delivery/27-i02-remediation.md new file mode 100644 index 0000000..2ba5ea0 --- /dev/null +++ b/docs/08-delivery/27-i02-remediation.md @@ -0,0 +1,48 @@ +# I-02H 整改任务与证据闭环验收 + +状态（2026-10-05）：本批代码和本地回归已完成，尚未提交、Push 或取得远程 CI/独立评审。report/export 不属于本批；生产环境仍禁止。 + +## 1. 交付范围 + +本批把确认风险接入可追踪的整改生命周期： + +- `dispatchRemediation`：仅 confirmed finding 可派发，锁定实验室、受派人和 due_at；原子写入 task、finding 状态、review action、audit、outbox 和通知。 +- `acceptRemediation`、`submitevidenceRemediation`：受派人按 task 版本认领并提交证据；图片必须是同一 remediation task 的 ready 图片。 +- `recheckRemediation`、`rejectRemediation`：独立复查人使用最新 evidence；受派人和 evidence submitter 均被排除。 +- `cannotremediateRemediation`、`reassignRemediation`：管理员无法整改收敛、具备 dispatch 权限的操作者转派回 pending_dispatch。 +- `listRemediations`、`getRemediation`、`listEvidence`：租户/实验室授权后的白名单投影，`overdue` 为查询派生值，不落库伪造状态。 + +所有写命令复用 CSRF、租户锁、幂等 claim、死锁有限重试和版本围栏。事件只在事务内写入 outbox；本批不宣称 publisher 已实际发布。 + +## 2. 关键不变量 + +| 场景 | 验收要求 | +| --- | --- | +| 派发 | finding 必须为 confirmed；finding 更新和 task 插入不能部分提交；同 key 重放返回原响应且不重复通知/审计/outbox | +| 证据 | image tenant、laboratory、remediation_task_id 必须与 task 一致，状态必须为 ready；插入 evidence、绑定图片、latest pointer 和状态更新同事务 | +| 复查 | 只能检查 task 的 latest evidence；assignee 或提交人执行返回稳定 `FORBIDDEN`；recheck 原子关闭 task/finding，reject 回到 rejected/dispatched | +| 并发 | expected_version 过期返回 `VERSION_CONFLICT`；task/finding 的 rowcount 都必须命中，否则整笔回滚 | +| 隔离 | 跨租户和跨实验室资源在授权前隐藏；HTTP 不返回数据库异常详情 | + +## 3. 验收证据 + +业务/领域/协议回归： + +```text +.venv-business\Scripts\python.exe -m pytest tests/protocol tests/domain tests/business -q +1477 passed (baseline) + +本批新增 API 边界 6 项通过，合计业务/领域/协议回归覆盖 1483 项。 +``` + +完整持久化回归使用专用 runner，创建随机回环端口、临时 data directory 和 disposable schema；不会连接或修改已有 MySQL。整改专项用例为 `tests/persistence/test_remediation_mysql.py`，覆盖派发原子写集和幂等、accept/submit/recheck 状态链、ready 图片归属及 stale version。 + +```text +.venv-business\Scripts\python.exe tools/database/run_mysql_tests.py --mysqld "D:\SQL\MySQL\MySQL Server 8.0\bin\mysqld.exe" +``` + +完整持久化基线为 MySQL 8.0.33、`0001_initial`、43 表、115 外键，既有 386 项通过；本批整改专项在同版本隔离 runner 中新增 2 项并通过（合计覆盖 388 项，完整套件最后一次执行时新增用例尚未纳入，故不把两者合并宣称为一次 388 项全绿运行）。 + +## 4. 未关闭项 + +真实对象存储/IAM/TLS、领域事件 publisher 的实际投递、report/export、完整 UI 和生产发布门禁仍未关闭。整改 API 的完成不替代这些阶段门禁。 diff --git a/docs/08-delivery/28-i02-report-export.md b/docs/08-delivery/28-i02-report-export.md new file mode 100644 index 0000000..4b10344 --- /dev/null +++ b/docs/08-delivery/28-i02-report-export.md @@ -0,0 +1,43 @@ +# I-02I1 report/export 快照与任务受理 + +状态（2026-10-05）：本批实现创建导出、状态查询及 report_export 管理员重放基础；仅 dev/test。本地验收见下表，未提交、Push 或创建 PR。report/export 整体尚未完成，不能关闭 EXPORT-01 或 JOB-04。 + +## 已交付 + +- `POST /api/v1/reports/exports`：严格 ExportCreate/ExportFilter、CSRF、幂等及限流，成功返回 202。逐实验室检查 export 权限；lab_manager 可导出其已授权范围，viewer 动作不足 403，跨租户/不可见实验室 404。 +- 31 天（含边界）、100 实验室、10000 条匹配的当前 finding 容量守卫；UTC 时间、重复 ID/筛选值拒绝；超量 422 提示缩小范围。日期按巡检创建时间筛选，from/to 两端包含；空 severity/status 表示不限。零 finding 巡检项仍保留一行，不占 finding 额度。 +- 同一个 REPEATABLE READ 事务冻结字段及关联 ID：仅当前 run/fact revision/evaluation 且未 superseded 的 finding；以 laboratory→inspection→item→finding 排序。列遵循 API 文档 5.3；不包含用户身份或原图 GPS。count 与行数据共用事务一致性视图，后续业务变更不会重写快照。 +- `report_exports.snapshot` 保存 schema_version、export_id、tenant_id、完整 laboratory_ids、snapshot_at、columns 和 rows。本批不新增表；文档中的 report snapshots 对应已有 JSON 列。 +- 原子写入 queued 导出、ready report_export task、TaskDispatch、ReportRequested、审计和 202 幂等响应。相同 key 返回原响应；重放仍检查当前全范围 export 权限，并使用当前读避免并发请求拿旧 RR 视图误报 404。 +- `GET /api/v1/reports/exports/{id}` 返回 ReportExport 白名单。对快照完整实验室范围重检 read 权限，包括没有匹配数据的实验室；不向客户端暴露 snapshot/object key。 +- 既有 getJob/listDeadLetters/replayJob 扩展 report_export；死信列表的 laboratory_id 匹配完整 filters 范围。重放只接受 admin、正确 task.version、failed/dead_letter task 与 failed export，核对 export/task/payload/snapshot 绑定，按 report→task 加锁。 +- 重放恢复 export queued；任务 generation、fencing_token、dispatch_sequence 递增，attempt 清零，清理旧 lease/错误/执行时间，保留原 snapshot/payload/旧 attempts，TaskDispatch/审计/幂等响应同事务。 + +## 验证 + +| 检查 | 本轮结果 | +|---|---| +| 业务、领域、协议 | 1500 项通过（含新增 17 项报告 HTTP/范围/消息守卫） | +| report/export MySQL 专项 | 最终 21 项通过（包含在下方 86 项内） | +| 既有完整持久化基线 | 首轮 388 项通过；该轮旧版新增测试夹具解包错误已修复，未将失败轮记为全量通过 | +| 最终受影响 MySQL 回归 | report_export、job API、dispatch 共 86 项全部通过，323 项未选择；不是全量 409 项通过 | +| 静态检查 | Ruff check、13 个相关文件 format --check、compileall、git diff --check 均通过 | + +MySQL 8.0.33 通过 `tools/database/run_mysql_tests.py` 启动全新临时 data directory、随机回环端口及独立 schema；不连接既有数据库。最终定向运行使用： + +```powershell +$env:PYTEST_ADDOPTS='-k "report_export or test_job_api_mysql or test_dispatch_mysql"' +.venv-business/Scripts/python.exe tools/database/run_mysql_tests.py --mysqld 'D:\SQL\MySQL\MySQL Server 8.0\bin\mysqld.exe' +``` + +真实测试覆盖当前关联/历史排除、零 finding 行、10000/10001 边界、实验室级权限、跨范围隐藏、原响应幂等/冲突/撤权、五类写入失败全回滚、错误版本/状态/绑定拒绝、failed 重放与旧 attempt 保留。 + +## 后续边界与回退 + +2026-10-06 续批：CSV 的发布、执行租约、编码、固定对象版本上传和 ready 登记已由 [I-02I2](29-i02-report-csv-execution.md) 接续实现；下述“下一批”描述为 I-02I1 当时边界，PDF/下载/通知等未实现部分仍保留。 + +本批仅登记任务意图。publisher 的任务类型白名单暂未开放 report_export，TaskDispatch 保持 pending，导出保持 queued，避免向尚不存在的消费者持续投递。ReportRequested 亦只写入 outbox，不宣称已实际投递。 + +下一批需接入 report 专属 lease/attempt/heartbeat/fencing、回收/失败收敛、CSV 公式防注入、PDF 分组及证据缩略图/人工意见快照扩展、对象版本登记和上传、ready/expiry、完成通知以及 downloadExport。当前没有可下载文件；CSV/PDF format 意图可受理，但不声明已生成。下载须满足创建人或同范围 admin、全范围 export 权限、24h 内及 60 秒签名；不能用未固定版本的对象 URL 提前实现。 + +回退恢复上一版程序并保留 report/task/outbox/audit/幂等历史；不清表、不删除对象或修改旧 attempt。真实存储/IAM/TLS、UI 与生产发布仍待独立验收。 diff --git a/docs/08-delivery/29-i02-report-csv-execution.md b/docs/08-delivery/29-i02-report-csv-execution.md new file mode 100644 index 0000000..b547808 --- /dev/null +++ b/docs/08-delivery/29-i02-report-csv-execution.md @@ -0,0 +1,45 @@ +# I-02I2 CSV 报告持久执行与对象版本登记 + +状态（2026-10-06）：本批接续 I-02I1 的冻结快照，完成 CSV 报告的 dev/test 执行链路。默认关闭，未提交、Push 或创建 PR；不关闭完整 EXPORT-01 / JOB-04。 + +## 本批交付 + +- report→task 专属锁序；消息 tenant/task/resource/payload/generation/sequence 只用于核对持久记录。首次/重试仅领取 queued CSV，原子创建 attempt 并切换 report running/task leased。 +- 60 秒数据库租约、独立 10 秒心跳、120 秒有界生成/上传子进程。外部工作不持数据库事务；完成后重新核对 report version、完整冻结输入和 owner/token/generation/attempt/有效租约。 +- CSV 仅取冻结列/行，不查当前业务数据。UTF-8 BOM、CRLF、标准 CSV 引号转义；以 =、+、-、@、tab、CR 开头的文本加单引号。保留零 finding 行及 UTC 快照时间；输出上限 64 MiB，错误快照/超限为不可重试 SCHEMA_MISMATCH。 +- 按排序最小 laboratory_id 构造规范 R 路径，文件名为实际 CSV SHA256。已有同 key 对象须读取精确版本并实算相同 SHA 才复用；不信任 ETag/声明 metadata，不同内容拒绝。PUT 必须返回有效、非 null VersionId。 +- 围栏事务原子登记 object_key/checksum/object_version/size_bytes、ready、expires_at=DB now+24h、系统 actor 的 report.export_ready 审计以及 task/attempt succeeded；不冒充申请用户，不伪造 ReportReady 事件。 +- DEPENDENCY_UNAVAILABLE/STAGE_TIMEOUT 按 5/30/120 秒加 0–20% 抖动重试；最多 4 attempts，耗尽进入 dead_letter/report failed。永久错误立即 failed。回收将旧 attempt 标 abandoned 并递增围栏；手工 replay 使用既有 API，保留快照和 attempt 历史。 +- `WORKER_REPORT_EXPORT_ENABLED=1` 在 publisher/Worker 显式启用 CSV。独立 q.reports 避免未启用报告的 general Worker 消费报告通知。publisher 回收/补发均按库中 csv format 过滤；PDF 保持 queued/pending。 +- 新迁移 `0002_report_object_version` 为 report_exports 追加两个 nullable 字段；原始迁移不变，生成源/设计契约/结构核对/readiness 同步。测试验证有旧快照的库升级后原字段不变。 + +## 验证 + +| 检查 | 本轮结果 | +|---|---| +| 业务/领域/安全/协议 | 1547 项全部通过，包含新增报告执行单元 33 项 | +| 报告执行与迁移定向验证 | 首轮修正结构检查器的旧 head 常量后，63 项通过（35 项报告 MySQL + 7 项结构检查 + 21 项持久化单元）；后续增加 2 项重放/旧上传围栏用例纳入完整回归 | +| 完整持久化回归 | 446 项全部通过（425 项真实 MySQL + 21 项持久化单元），包含报告执行 37 项及既有报告受理 21 项；MySQL 8.0.33、head=0002_report_object_version、43 表/115 外键 | +| 静态检查 | Ruff check、176 文件 format --check、git diff --check 通过 | +| 设计契约 | build_specs --check、148 项 readiness 设计检查、validate_specs 全部通过 | + +报告专项覆盖并发领取、重复/旧/未来消息、8 类围栏或输入变化、4 次租约回收、10 个写入故障点回滚、事务外生成、CSV/PDF 发布门禁、失败重放和旧上传结果不能覆盖新结果。新增测试文件合计 70 项（33 单元 + 37 MySQL），不与上表重复累加。 + +真实 MySQL 验证使用 `tools/database/run_mysql_tests.py`，仅创建独立临时 data directory/schema 和随机回环端口；不连接既有数据库。S3 使用真实 SDK Stubber，不能替代实际 MinIO/IAM/TLS 验收。 + +完整回归命令（不设置 PYTEST_ADDOPTS 筛选）： + +```powershell +.venv-business/Scripts/python.exe -m pytest tests/business tests/domain tests/security tests/protocol -q +.venv-business/Scripts/python.exe tools/database/run_mysql_tests.py --mysqld 'D:/SQL/MySQL/MySQL Server 8.0/bin/mysqld.exe' +``` + +完整持久化回归耗时 21 分 54 秒，无失败或跳过；仅有现有 Starlette/AnyIO 弃用警告。首次定向运行因结构检查器仍要求旧 head 导致夹具失败，已同步修正并完成上述重跑，未将失败轮算作通过。 + +## 启动、后续与回退 + +启动方式及 secret 配置见 [开发指南](../../DEVELOPMENT.md)。先迁移并 verify，再分别运行 publisher/sweeper/报告 Worker。已有旧版数据库需升级后才能通过 readiness。 + +后批仍需：PDF 所需授权证据/人工意见快照扩展和渲染；完成通知；ready 到 expired 的过期处理与精确对象版本清理；真实存储部署及 UI。ReportRequested 仍只是领域 outbox 记录。I-02I3 已补齐 CSV `downloadExport`：创建人或同范围 admin、全范围 export 权限、24h 内及 60 秒精确版本签名，见[报告下载验收](30-i02-report-download.md)。 + +停止报告 publisher 并排空消费者，再关闭报告开关；保留 schema、对象与任务/audit 历史。上传成功后 DB 未提交或旧租约丢失可能留下未引用版本，本批不删除它们。未来清理须查 DB 引用并按精确 key+version 执行，禁止按前缀递归删除；有数据时不以 downgrade 回退。 diff --git a/docs/08-delivery/30-i02-report-download.md b/docs/08-delivery/30-i02-report-download.md new file mode 100644 index 0000000..23d5ec8 --- /dev/null +++ b/docs/08-delivery/30-i02-report-download.md @@ -0,0 +1,22 @@ +# I-02I3 报告下载签发验收 + +日期：2026-10-07。范围：接通既有 `downloadExport` 契约，为已完成的 CSV 报告签发受控 GET URL。PDF、完成通知、过期状态转换、对象清理和真实对象存储联调仍属于后续边界。 + +## 契约与授权 + +- `GET /api/v1/reports/exports/{id}/download` 不接受 query 参数，返回既有 `DownloadGrantResponse`，响应使用 `no-store` 和 `no-referrer`。 +- 只有 `ready`、CSV、`expires_at` 仍在窗口内且具备完整固定对象证据的报告可下载；queued/running/failed/expired、过期或缺少 key/checksum/VersionId/size 均拒绝且不签名。 +- 请求人在报告创建人或对完整快照范围具有 admin 权限时才可下载，并且每个快照实验室都必须重新具备 `export` 权限。跨租户或不可见范围隐藏为 404，部分范围授权不能下载。 +- 下载只使用冻结报告登记的 checksum、规范对象 key 和准确 VersionId；不重新查询业务数据、不重新生成文件、不向响应或审计暴露内部对象字段。 + +## 实现边界 + +`packages/domain/report_download.py` 校验 CSV key、SHA、对象版本、大小和范围；`packages/persistence/reports.py` 在租户事务中锁定并读取私有证据；`packages/application/reports.py` 在事务外执行共享 download grant 限流，在租户/报告锁内重新认证授权后调用本地 S3 SigV4；`packages/storage/s3.py` 的 `presign_report_get` 固定 GET、公共 origin、准确 VersionId 和 60 秒 TTL，并复核完整签名查询。 + +数据库事务不执行 HEAD/GET/PUT。签名失败、寿命异常或证据不一致不会返回 URL，也不改变报告状态。 + +## 验证 + +业务测试见 [test_report_download.py](../../tests/business/test_report_download.py)，MySQL/HTTP 专项见 [test_report_download_mysql.py](../../tests/persistence/test_report_download_mysql.py)。覆盖规范引用、非 ready/过期/缺失或错误证据、创建人和同范围 admin、撤权与部分实验室、跨租户 404、固定 60 秒 TTL、精确 VersionId 传入 signer、query 拒绝及不触发重新生成。业务回归 1550 项、此前完整持久化回归 446 项通过；新增 MySQL 专项随完整 runner 执行，真实 S3/IAM/TLS 联调仍需部署验收。 + +2026-10-07 提交前重验：业务/领域/安全/协议1550项通过。隔离MySQL8.0.33 runner以 `PYTEST_ADDOPTS=-k "report or remediation or structure"` 筛选当前447项中的73项，全部通过、374项未选中；包含新增报告下载及整改/报告受理/执行用例，迁移head=0002_report_object_version、43表/115外键。此为定向回归，不宣称本次重新执行完整447项；历史446项完整成绩仍按第29批记录。 diff --git a/docs/08-delivery/README.md b/docs/08-delivery/README.md index 9e05f3d..4e21a23 100644 --- a/docs/08-delivery/README.md +++ b/docs/08-delivery/README.md @@ -26,5 +26,9 @@ 24. [I-03A2规则评估任务执行](24-i03a2-rule-evaluation.md)。 25. [I-02规则配置管理](25-i02-rule-management.md)。 26. [I-02事实修订与人工复核](26-i02-review.md)。 +27. [I-02整改任务与证据闭环](27-i02-remediation.md)。 +28. [I-02I1 report/export 快照与任务受理](28-i02-report-export.md)。 +29. [I-02I2 CSV 报告持久执行与对象版本登记](29-i02-report-csv-execution.md)。 +30. [I-02I3 报告下载签发](30-i02-report-download.md)。 本页仅导航；现行设计状态见 [文档控制](../00-document-control.md)，不把候选设计视为已经批准。 diff --git a/packages/application/dispatch.py b/packages/application/dispatch.py index 1edcf5d..546fa30 100644 --- a/packages/application/dispatch.py +++ b/packages/application/dispatch.py @@ -19,9 +19,10 @@ def transaction(engine): class DispatchPublisher: - def __init__(self, engine, transport): + def __init__(self, engine, transport, *, report_enabled=False): self.engine = engine self.transport = transport + self.report_enabled = report_enabled def publish_batch(self, limit=100): if type(limit) is not int or not 1 <= limit <= 100: @@ -30,7 +31,7 @@ def publish_batch(self, limit=100): for _ in range(limit): owner = str(uuid4()) with transaction(self.engine) as connection: - row = dispatch.claim(connection, owner) + row = dispatch.claim(connection, owner, report_enabled=self.report_enabled) if row is None: break @@ -62,9 +63,9 @@ def renew(): return published -def sweep_dispatch(engine): +def sweep_dispatch(engine, *, report_enabled=False): with transaction(engine) as connection: - recovered = dispatch.recover_publications(connection) + recovered = dispatch.recover_publications(connection, report_enabled=report_enabled) with transaction(engine) as connection: - resent = dispatch.redispatch(connection) + resent = dispatch.redispatch(connection, report_enabled=report_enabled) return recovered, resent diff --git a/packages/application/identity.py b/packages/application/identity.py index 75b7c2e..76ed1f1 100644 --- a/packages/application/identity.py +++ b/packages/application/identity.py @@ -88,7 +88,7 @@ def readiness(self): # This is identity readiness, not the future object/AI/worker readiness gate. with self.engine.connect() as connection: revision = connection.scalar(text("SELECT version_num FROM alembic_version")) - if revision != "0001_initial": + if revision != "0002_report_object_version": raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Database revision is not ready") self.limits.ready() diff --git a/packages/application/jobs.py b/packages/application/jobs.py index 93938c6..15d9161 100644 --- a/packages/application/jobs.py +++ b/packages/application/jobs.py @@ -1,4 +1,4 @@ -"""Manual image-job replay: authenticate, observe fixed input outside locks, revalidate, commit.""" +"""Manual replay; image inputs are inspected outside locks, report snapshots stay in DB.""" from datetime import timedelta from uuid import uuid4 @@ -89,11 +89,11 @@ def _replay_once(self, token, csrf, key, job_id, body, request_id, *, observed=N return claim.response_status, claim.response task, current = self.jobs.replay_source(connection, actor, job_id) require_replay(actor, task, current, body["expected_version"]) - if observed is None: + if task["task_type"] != "report_export" and observed is None: raise _NeedsPinnedInspection( tenant, current ) # Rolls back provisional claim and locks. - if current != observed: + if task["task_type"] != "report_export" and current != observed: raise ServiceError("STATE_CONFLICT", 409, "Pinned input changed during inspection") data = self.jobs.replay(connection, actor, task, body["reason"], request_id) response = {"data": data, "request_id": request_id} diff --git a/packages/application/remediation.py b/packages/application/remediation.py new file mode 100644 index 0000000..7bdc9e4 --- /dev/null +++ b/packages/application/remediation.py @@ -0,0 +1,143 @@ +"""Application transactions for remediation commands and evidence.""" + +from datetime import datetime, timedelta +from uuid import UUID, uuid4 + +from packages.application.identity import retry_deadlocks +from packages.domain.security import ServiceError +from packages.persistence import remediation as repository +from packages.persistence.security import begin_idempotency, complete_idempotency, utc_now +from packages.persistence.transaction import lock_tenant + + +class RemediationApplication: + def __init__(self, identity): + self.identity = identity + + def read( + self, + kind, + token, + request_id, + *, + resource_id=None, + laboratory_id=None, + page=1, + page_size=20, + status=None, + overdue=None, + ): + actor = self.identity._preflight(token) + self.identity.limits.user(actor, write=False) + return self._read_once( + kind, + token, + request_id, + resource_id, + laboratory_id, + page, + page_size, + status, + overdue, + ) + + @retry_deadlocks + def _read_once( + self, + kind, + token, + request_id, + resource_id, + laboratory_id, + page, + page_size, + status, + overdue, + ): + with self.identity.transaction() as connection: + actor = self.identity.sessions.authenticate(connection, token).principal + if kind == "task": + data = repository.get_task(connection, actor, resource_id) + elif kind == "tasks": + data = repository.list_tasks( + connection, actor, page, page_size, laboratory_id, status, overdue + ) + elif kind == "evidence": + data = repository.list_evidence( + connection, actor, resource_id, page, page_size, laboratory_id + ) + else: + raise ServiceError("VALIDATION_ERROR", 422, "Unknown remediation query") + return {"data": data, "request_id": request_id} + + def write(self, operation, token, csrf, key, resource_id, body, request_id): + actor = self.identity._preflight(token, csrf=csrf) + self.identity.limits.user(actor, write=True) + try: + resource_id = str(UUID(resource_id)) + except (TypeError, ValueError): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid resource id") from None + if operation not in { + "dispatch", + "accept", + "submit-evidence", + "recheck", + "reject", + "cannot-remediate", + "reassign", + }: + raise ServiceError("VALIDATION_ERROR", 422, "Unknown remediation command") + return self._write_once(operation, token, csrf, key, resource_id, body, request_id) + + @retry_deadlocks + def _write_once(self, operation, token, csrf, key, resource_id, body, request_id): + path = ( + f"/api/v1/findings/{resource_id}/remediation-tasks" + if operation == "dispatch" + else f"/api/v1/remediation-tasks/{resource_id}/{operation}" + ) + with self.identity.transaction() as connection: + tenant, actor_id = self.identity.sessions.locate(connection, token) + lock_tenant(connection, tenant) + owner = str(uuid4()) + claim = begin_idempotency( + connection, + tenant_id=tenant, + actor_id=actor_id, + method="POST", + path=path, + key=key, + request_hash=self.identity.sessions.request_digest(body), + expires_at=utc_now() + timedelta(hours=24), + lease_owner=owner, + ) + session = self.identity.sessions.authenticate(connection, token) + self.identity.sessions.require_csrf(session, csrf) + actor = session.principal + if operation in {"dispatch", "reassign"} and isinstance(body.get("due_at"), str): + body = { + **body, + "due_at": datetime.fromisoformat(body["due_at"].replace("Z", "+00:00")), + } + if claim.state == "completed": + if operation == "dispatch": + repository.authorize_finding(connection, actor, resource_id) + else: + repository.get_task(connection, actor, resource_id) + return claim.response_status, claim.response + if operation == "dispatch": + data = repository.dispatch(connection, actor, resource_id, body, request_id) + status = 201 + elif operation == "submit-evidence": + data = repository.submit(connection, actor, resource_id, body, request_id) + status = 200 + else: + data = repository.command( + connection, actor, resource_id, body, operation, request_id + ) + status = 200 + response = {"data": data, "request_id": request_id} + complete_idempotency( + connection, claim, status=status, response=response, lease_owner=owner + ) + return status, response diff --git a/packages/application/report_execution.py b/packages/application/report_execution.py new file mode 100644 index 0000000..d59c4b5 --- /dev/null +++ b/packages/application/report_execution.py @@ -0,0 +1,62 @@ +"""Execute frozen CSV reports with independent heartbeat and fenced result writes.""" + +from threading import Event +from uuid import uuid4 + +from packages.application.dispatch import transaction +from packages.application.job_execution import LeaseHeartbeat +from packages.domain.job_execution import LeaseLost +from packages.domain.report_execution import REPORT_ERRORS, ReportInvalid +from packages.domain.security import ServiceError +from packages.persistence import report_execution +from packages.shared.environment import development_environment + + +class ReportExecution: + def __init__(self, engine): + development_environment() + self.engine = engine + + def claim(self, message): + with transaction(self.engine) as connection: + return report_execution.claim(connection, message, str(uuid4())) + + def heartbeat(self, lease): + with transaction(self.engine) as connection: + return report_execution.heartbeat(connection, lease) + + def fail(self, lease, code): + with transaction(self.engine) as connection: + return report_execution.fail(connection, lease, code) + + def commit(self, lease, artifact): + with transaction(self.engine) as connection: + return report_execution.commit_result(connection, lease, artifact) + + def execute(self, message, prepare): + lease = self.claim(message) + if lease is None: + return False + cancelled = Event() + try: + with LeaseHeartbeat(self, lease, cancelled.set): + artifact = prepare(lease.input, cancelled) + self.commit(lease, artifact) + except LeaseLost: + raise + except Exception as error: + code = error.code if isinstance(error, ServiceError) else "INTERNAL_ERROR" + if isinstance(error, ReportInvalid): + code = "SCHEMA_MISMATCH" + self.fail(lease, code if code in REPORT_ERRORS else "INTERNAL_ERROR") + raise + return True + + def recover_expired(self, limit=100): + with transaction(self.engine) as connection: + candidates = report_execution.expired_candidates(connection, limit) + recovered = 0 + for candidate in candidates: + with transaction(self.engine) as connection: + recovered += int(report_execution.recover(connection, candidate)) + return recovered diff --git a/packages/application/reports.py b/packages/application/reports.py new file mode 100644 index 0000000..86072da --- /dev/null +++ b/packages/application/reports.py @@ -0,0 +1,99 @@ +"""Report snapshot acceptance; all writes commit with the idempotent response.""" + +from datetime import timedelta +from uuid import uuid4 + +from packages.application.identity import retry_deadlocks +from packages.domain.report_download import REPORT_DOWNLOAD_SECONDS, download_reference +from packages.domain.security import Permission, ServiceError +from packages.persistence import reports +from packages.persistence.dispatch import decoded +from packages.persistence.security import begin_idempotency, complete_idempotency, utc_now +from packages.persistence.transaction import lock_tenant +from packages.persistence.users import timestamp + + +class ReportApplication: + def __init__(self, identity, storage=None): + self.identity, self.storage = identity, storage + + def create(self, token, csrf, key, body, request_id): + actor = self.identity._preflight(token, csrf=csrf) + self.identity.limits.user(actor, write=True) + return self._create_once(token, csrf, key, body, request_id) + + @retry_deadlocks + def _create_once(self, token, csrf, key, body, request_id): + with self.identity.transaction() as connection: + tenant, actor_id = self.identity.sessions.locate(connection, token) + lock_tenant(connection, tenant) + owner = str(uuid4()) + claim = begin_idempotency( + connection, + tenant_id=tenant, + actor_id=actor_id, + method="POST", + path="/api/v1/reports/exports", + key=key, + request_hash=self.identity.sessions.request_digest(body), + expires_at=utc_now() + timedelta(hours=24), + lease_owner=owner, + ) + session = self.identity.sessions.authenticate(connection, token) + self.identity.sessions.require_csrf(session, csrf) + if claim.state == "completed": + reports.get( + connection, + session.principal, + claim.response["data"]["id"], + permission=Permission.EXPORT, + current=True, + ) + return claim.response_status, claim.response + data = reports.create(connection, session.principal, body, request_id) + response = {"data": data, "request_id": request_id} + complete_idempotency( + connection, claim, status=202, response=response, lease_owner=owner + ) + return 202, response + + def read(self, token, request_id, export_id): + actor = self.identity._preflight(token) + self.identity.limits.user(actor, write=False) + return self._read_once(token, request_id, export_id) + + @retry_deadlocks + def _read_once(self, token, request_id, export_id): + with self.identity.transaction() as connection: + actor = self.identity.sessions.authenticate(connection, token).principal + return {"data": reports.get(connection, actor, export_id), "request_id": request_id} + + def download(self, token, export_id, request_id): + if self.storage is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Report downloads not configured") + actor = self.identity._preflight(token) + self.identity.limits.download_grant(actor) + return self._download_once(token, export_id, request_id) + + @retry_deadlocks + def _download_once(self, token, export_id, request_id): + with self.identity.transaction() as connection: + actor = self.identity.sessions.authenticate(connection, token).principal + row = reports.download_row(connection, actor, export_id) + if row is not None: + try: + labs = decoded(row["filters"])["laboratory_ids"] + except (TypeError, ValueError, KeyError): + labs = None + if isinstance(labs, list) and all(isinstance(lab, str) for lab in labs): + # Full snapshot scope is required even for a creator/admin. + reports.authorize_labs(connection, actor, labs, Permission.EXPORT) + reference = download_reference(actor, row, utc_now()) + grant = self.storage.presign_report_get(reference) + remaining = (grant.expires_at - utc_now()).total_seconds() + if not 0 < remaining <= REPORT_DOWNLOAD_SECONDS: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Invalid download lifetime") + return { + "data": {"url": grant.url, "expires_at": timestamp(grant.expires_at)}, + "request_id": request_id, + } diff --git a/packages/domain/job_replay.py b/packages/domain/job_replay.py index 0a07bf8..50ff014 100644 --- a/packages/domain/job_replay.py +++ b/packages/domain/job_replay.py @@ -1,4 +1,4 @@ -"""Manual validate_image replay guards; no queue or object calls.""" +"""Manual image/report task replay guards; no queue or object calls.""" from packages.domain.security import Permission, ServiceError, authorize, not_found, require_version @@ -20,7 +20,7 @@ def require_replay(actor, task, current, expected_version): if ( task is None or task["tenant_id"] != actor.tenant_id - or task["task_type"] != "validate_image" + or task["task_type"] not in {"validate_image", "report_export"} ): raise not_found() require_version(task["version"], expected_version) diff --git a/packages/domain/report_download.py b/packages/domain/report_download.py new file mode 100644 index 0000000..9aeb136 --- /dev/null +++ b/packages/domain/report_download.py @@ -0,0 +1,92 @@ +"""Authorized, fixed-version report download references.""" + +from dataclasses import dataclass + +from packages.domain.report_execution import MAX_REPORT_BYTES, REPORT_MIME +from packages.domain.security import Permission, ServiceError, authorize, not_found +from packages.domain.uploads import digest, identifier + +REPORT_DOWNLOAD_SECONDS = 60 + + +@dataclass(frozen=True) +class ReportDownload: + tenant_id: str + export_id: str + laboratory_ids: tuple[str, ...] + checksum: str + key: str + object_version: str + size_bytes: int + mime_type: str + + +def validate_reference(reference: ReportDownload) -> None: + if not isinstance(reference, ReportDownload): + raise ValueError("Invalid report download reference") + identifier(reference.tenant_id) + identifier(reference.export_id) + labs = reference.laboratory_ids + if not isinstance(labs, tuple) or not 1 <= len(labs) <= 100 or tuple(sorted(set(labs))) != labs: + raise ValueError("Invalid report scope") + for lab in labs: + identifier(lab) + digest(reference.checksum) + expected = ( + f"tenant/{reference.tenant_id}/lab/{labs[0]}/reports/" + f"{reference.export_id}/{reference.checksum}.csv" + ) + if reference.key != expected: + raise ValueError("Invalid report download key") + version = reference.object_version + if ( + not isinstance(version, str) + or not 1 <= len(version) <= 200 + or version == "null" + or any(not 33 <= ord(c) <= 126 for c in version) + ): + raise ValueError("Invalid report download version") + if type(reference.size_bytes) is not int or not 1 <= reference.size_bytes <= MAX_REPORT_BYTES: + raise ValueError("Invalid report download size") + if reference.mime_type != REPORT_MIME: + raise ValueError("Invalid report download media type") + + +def download_reference(actor, row, now): + if row is None or row["tenant_id"] != actor.tenant_id: + raise not_found() + try: + import json + + filters = json.loads(row["filters"]) if isinstance(row["filters"], str) else row["filters"] + labs = filters["laboratory_ids"] + except (TypeError, ValueError, KeyError): + raise ServiceError( + "STATE_CONFLICT", 409, "Report download evidence is incomplete" + ) from None + if not isinstance(labs, list) or any(not isinstance(lab, str) for lab in labs): + raise ServiceError("STATE_CONFLICT", 409, "Report download evidence is incomplete") + if actor.user_id != row["requested_by"]: + for lab in labs: + authorize(actor, Permission.ADMIN, lab) + if row["status"] != "ready" or row["format"] != "csv": + raise ServiceError("STATE_CONFLICT", 409, "Report is not ready for download") + if row["expires_at"] is None or row["expires_at"] <= now: + raise ServiceError("STATE_CONFLICT", 409, "Report download has expired") + reference = ReportDownload( + row["tenant_id"], + row["id"], + tuple(labs), + row["checksum"], + row["object_key"], + row["object_version"], + row["size_bytes"], + REPORT_MIME, + ) + try: + validate_reference(reference) + except (ValueError, TypeError): + raise ServiceError( + "STATE_CONFLICT", 409, "Report download evidence is incomplete" + ) from None + return reference diff --git a/packages/domain/report_execution.py b/packages/domain/report_execution.py new file mode 100644 index 0000000..711e866 --- /dev/null +++ b/packages/domain/report_execution.py @@ -0,0 +1,167 @@ +"""Immutable report execution values and deterministic, bounded CSV encoding.""" + +import csv +import hashlib +import io +import json +from dataclasses import dataclass +from itertools import chain + +from packages.domain.job_execution import LeaseLost +from packages.domain.reports import COLUMNS, MAX_FINDINGS +from packages.domain.uploads import capture_time, digest, identifier + +MAX_REPORT_BYTES = 64 * 1024 * 1024 +REPORT_SECONDS = 120 +REPORT_MIME = "text/csv; charset=utf-8" +REPORT_RETRYABLE = {"DEPENDENCY_UNAVAILABLE", "STAGE_TIMEOUT"} +REPORT_ERRORS = REPORT_RETRYABLE | {"INTERNAL_ERROR", "SCHEMA_MISMATCH"} + + +class ReportLeaseLost(LeaseLost): + pass + + +class ReportInvalid(ValueError): + def __init__(self): + super().__init__("Invalid frozen report snapshot") + + +@dataclass(frozen=True) +class ReportInput: + export_id: str + tenant_id: str + requested_by: str + format: str + filters: str + snapshot_at: str + snapshot: str + version: int + + +@dataclass(frozen=True) +class ReportLease: + tenant_id: str + task_id: str + owner: str + token: int + generation: int + attempt: int + input: ReportInput + + +@dataclass(frozen=True) +class ReportArtifact: + key: str + checksum: str + object_version: str + size_bytes: int + + +def frozen_snapshot(source): + try: + identifier(source.tenant_id) + identifier(source.export_id) + value, filters = json.loads(source.snapshot), json.loads(source.filters) + labs = filters["laboratory_ids"] + if ( + source.format != "csv" + or not isinstance(labs, list) + or not 1 <= len(labs) <= 100 + or any(not isinstance(lab, str) for lab in labs) + or labs != sorted(set(labs)) + or not isinstance(value, dict) + or set(value) + != { + "schema_version", + "export_id", + "tenant_id", + "laboratory_ids", + "snapshot_at", + "columns", + "rows", + } + or value["schema_version"] != "1" + or value["tenant_id"] != source.tenant_id + or value["export_id"] != source.export_id + or value["laboratory_ids"] != labs + or value["snapshot_at"] != source.snapshot_at + or value["columns"] != list(COLUMNS) + or not isinstance(value["rows"], list) + ): + raise ReportInvalid() + capture_time(source.snapshot_at) + for lab in labs: + identifier(lab) + findings = 0 + for row in value["rows"]: + if ( + not isinstance(row, dict) + or set(row) != set(COLUMNS) + or any(v is not None and not isinstance(v, str) for v in row.values()) + or row["laboratory_id"] not in labs + or row["snapshot_at"] != source.snapshot_at + ): + raise ReportInvalid() + findings += row["finding_id"] is not None + if findings > MAX_FINDINGS: + raise ReportInvalid() + return value + except Exception as error: + if isinstance(error, ReportInvalid): + raise + raise ReportInvalid() from None + + +def csv_bytes(source, checkpoint=lambda: None): + snapshot = frozen_snapshot(source) + output = io.BytesIO() + output.write(b"\xef\xbb\xbf") + line = io.StringIO(newline="") + writer = csv.writer(line, lineterminator="\r\n") + for row in chain((COLUMNS,), (tuple(r[c] for c in COLUMNS) for r in snapshot["rows"])): + checkpoint() + line.seek(0) + line.truncate(0) + writer.writerow( + [ + "'" + v + if isinstance(v, str) and v.startswith(("=", "+", "-", "@", "\t", "\r")) + else v + for v in row + ] + ) + encoded = line.getvalue().encode("utf-8") + if output.tell() + len(encoded) > MAX_REPORT_BYTES: + raise ReportInvalid() + output.write(encoded) + return output.getvalue() + + +def report_key(source, checksum): + snapshot = frozen_snapshot(source) + digest(checksum) + return ( + f"tenant/{source.tenant_id}/lab/{snapshot['laboratory_ids'][0]}/" + f"reports/{source.export_id}/{checksum}.csv" + ) + + +def artifact_for(source, payload, object_version): + checksum = hashlib.sha256(payload).hexdigest() + result = ReportArtifact(report_key(source, checksum), checksum, object_version, len(payload)) + validate_artifact(source, result) + return result + + +def validate_artifact(source, artifact): + if ( + artifact.key != report_key(source, artifact.checksum) + or not isinstance(artifact.object_version, str) + or not 1 <= len(artifact.object_version) <= 200 + or artifact.object_version == "null" + or any(not 33 <= ord(c) <= 126 for c in artifact.object_version) + or type(artifact.size_bytes) is not int + or not 1 <= artifact.size_bytes <= MAX_REPORT_BYTES + ): + raise ReportInvalid() diff --git a/packages/domain/reports.py b/packages/domain/reports.py new file mode 100644 index 0000000..1f7376c --- /dev/null +++ b/packages/domain/reports.py @@ -0,0 +1,75 @@ +"""Export request bounds and fixed CSV snapshot columns.""" + +from datetime import timedelta + +from packages.domain.security import ServiceError +from packages.domain.uploads import capture_time, identifier + +MAX_FINDINGS = 10000 +COLUMNS = ( + "inspection_id", + "item_id", + "laboratory_id", + "location_id", + "run_id", + "fact_revision_id", + "rule_bundle_id", + "finding_id", + "rule_id", + "severity", + "status", + "explanation", + "task_status", + "review_outcome", + "snapshot_at", +) + + +def validate_request(body): + def invalid(): + return ServiceError("VALIDATION_ERROR", 422, "Invalid export range; narrow the filters") + + if not isinstance(body, dict) or set(body) != {"format", "filters"}: + raise invalid() + if body["format"] not in ("csv", "pdf"): + raise invalid() + filters = body["filters"] + if not isinstance(filters, dict) or set(filters) != { + "laboratory_ids", + "from", + "to", + "severity", + "finding_status", + }: + raise invalid() + labs = filters["laboratory_ids"] + if not isinstance(labs, list) or not 1 <= len(labs) <= 100: + raise invalid() + for lab in labs: + identifier(lab) + if len(set(labs)) != len(labs): + raise invalid() + for name, allowed in ( + ("severity", {"low", "medium", "high", "critical"}), + ( + "finding_status", + {"needs_review", "confirmed", "rejected", "cannot_determine", "dispatched", "closed"}, + ), + ): + values = filters[name] + if not isinstance(values, list) or len(values) > len(allowed): + raise invalid() + if any(not isinstance(value, str) or value not in allowed for value in values): + raise invalid() + if len(set(values)) != len(values): + raise invalid() + start, end = capture_time(filters["from"]), capture_time(filters["to"]) + if end < start or end - start > timedelta(days=31): + raise invalid() + normalized = { + **filters, + "laboratory_ids": sorted(labs), + "from": start.isoformat(timespec="milliseconds") + "Z", + "to": end.isoformat(timespec="milliseconds") + "Z", + } + return normalized, start, end diff --git a/packages/persistence/dispatch.py b/packages/persistence/dispatch.py index 0055cd2..dbd7605 100644 --- a/packages/persistence/dispatch.py +++ b/packages/persistence/dispatch.py @@ -39,6 +39,7 @@ def validate_dispatch(message): "validate_image", "inference_pipeline", "rule_evaluation", + "report_export", }: raise ValueError("Unsupported dispatch protocol or task type") for name in ("tenant_id", "task_id", "resource_id"): @@ -79,13 +80,17 @@ def validate_dispatch(message): or not 1 <= payload["submission_revision"] <= 2147483647 ): raise ValueError("Invalid inference revision") - else: + elif message["task_type"] == "rule_evaluation": if set(payload) != {"run_id", "fact_revision_id", "rule_bundle_id"}: raise ValueError("Invalid rule_evaluation payload") for name in ("run_id", "fact_revision_id", "rule_bundle_id"): identifier(payload[name]) if payload["run_id"] != message["resource_id"]: raise ValueError("Inconsistent rule evaluation resource") + else: + if set(payload) != {"export_id"} or payload["export_id"] != message["resource_id"]: + raise ValueError("Invalid report_export payload") + identifier(payload["export_id"]) return message @@ -98,16 +103,39 @@ def db_now(connection): return connection.scalar(text("SELECT UTC_TIMESTAMP(3)")) -def claim(connection, owner): +def _supported(alias, report_enabled, *, outbox=False): + if type(report_enabled) is not bool: + raise ValueError("Report dispatch flag must be boolean") + kind = ( + f"JSON_UNQUOTE(JSON_EXTRACT({alias}.payload,'$.task_type'))" + if outbox + else f"{alias}.task_type" + ) + existing = f"{kind} IN ('validate_image','inference_pipeline','rule_evaluation')" + if not report_enabled: + return existing + resource = ( + f"JSON_UNQUOTE(JSON_EXTRACT({alias}.payload,'$.resource_id'))" + if outbox + else f"{alias}.resource_id" + ) + return ( + f"({existing} OR ({kind}='report_export' AND EXISTS (" + f"SELECT 1 FROM report_exports e WHERE e.tenant_id={alias}.tenant_id " + f"AND e.id={resource} AND e.format='csv')))" + ) + + +def claim(connection, owner, *, report_enabled=False): """Claim just one row so waiting behind slow publishes cannot expire a batch.""" identifier(owner) db_now(connection) row = ( connection.execute( text( - "SELECT * FROM outbox_events WHERE event_type='TaskDispatch' " - "AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.task_type')) " - "IN ('validate_image','inference_pipeline','rule_evaluation') " + "SELECT o.* FROM outbox_events o WHERE event_type='TaskDispatch' AND " + + _supported("o", report_enabled, outbox=True) + + " " "AND state='pending' AND available_at<=UTC_TIMESTAMP(3) " "ORDER BY available_at,id LIMIT 1 FOR UPDATE SKIP LOCKED" ) @@ -172,16 +200,16 @@ def owned_update(connection, row, owner, operation): return True -def recover_publications(connection, limit=100): +def recover_publications(connection, limit=100, *, report_enabled=False): require_transaction(connection) if type(limit) is not int or not 1 <= limit <= 100: raise ValueError("Batch limit must be 1..100") rows = ( connection.execute( text( - "SELECT id FROM outbox_events WHERE event_type='TaskDispatch' " - "AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.task_type')) " - "IN ('validate_image','inference_pipeline','rule_evaluation') " + "SELECT o.id FROM outbox_events o WHERE event_type='TaskDispatch' AND " + + _supported("o", report_enabled, outbox=True) + + " " "AND state='leased' AND lease_until<=UTC_TIMESTAMP(3) " "ORDER BY lease_until,id LIMIT :limit FOR UPDATE SKIP LOCKED" ), @@ -202,7 +230,7 @@ def recover_publications(connection, limit=100): return len(rows) -def redispatch(connection, limit=100): +def redispatch(connection, limit=100, *, report_enabled=False): """Rebuild notices from DB payload, even if the initial event was published. last_dispatched_at is the time an intent was registered, not a broker ack. @@ -214,8 +242,7 @@ def redispatch(connection, limit=100): rows = ( connection.execute( text( - "SELECT * FROM task_runs WHERE task_type IN " - "('validate_image','inference_pipeline','rule_evaluation') " + "SELECT t.* FROM task_runs t WHERE " + _supported("t", report_enabled) + " " "AND state IN ('ready','retry_wait') AND available_at<=UTC_TIMESTAMP(3) " "AND (last_dispatched_at IS NULL OR " "last_dispatched_at<=UTC_TIMESTAMP(3)-INTERVAL 30 SECOND) " diff --git a/packages/persistence/jobs.py b/packages/persistence/jobs.py index 137404e..b60b2ae 100644 --- a/packages/persistence/jobs.py +++ b/packages/persistence/jobs.py @@ -1,11 +1,11 @@ -"""Image-job read models and atomic generation replay, using domain-first locks.""" +"""Image/report job read models and atomic replay, using domain-first locks.""" from uuid import UUID, uuid4 from sqlalchemy import text from packages.domain.security import Permission, authorize, not_found -from packages.persistence import job_execution +from packages.persistence import job_execution, reports from packages.persistence.dispatch import db_now, decoded, validate_dispatch from packages.persistence.foundations import BASE, project from packages.persistence.security import append_audit @@ -17,8 +17,10 @@ ) COLUMNS = ",".join("job." + name for name in FIELDS.split(",")) FROM = ( - "task_runs job JOIN asset_images image ON image.tenant_id=job.tenant_id " - "AND image.id=job.resource_id" + "task_runs job LEFT JOIN asset_images image ON image.tenant_id=job.tenant_id " + "AND image.id=job.resource_id AND job.task_type='validate_image' " + "LEFT JOIN report_exports report ON report.tenant_id=job.tenant_id " + "AND report.id=job.resource_id AND job.task_type='report_export'" ) @@ -30,11 +32,27 @@ def job_projection(row): class JobRepository: def get(self, connection, actor, job_id): + report = ( + connection.execute( + text( + "SELECT job.* " + "FROM task_runs job JOIN report_exports e ON e.tenant_id=job.tenant_id " + "AND e.id=job.resource_id " + "WHERE job.tenant_id=:tenant AND job.id=:id AND job.task_type='report_export'" + ), + {"tenant": actor.tenant_id, "id": job_id}, + ) + .mappings() + .first() + ) + if report is not None: + reports.get(connection, actor, report["resource_id"]) + return job_projection(report) row = ( connection.execute( text( f"SELECT {COLUMNS},image.laboratory_id FROM {FROM} WHERE job.tenant_id=:tenant " - "AND job.id=:id AND job.task_type='validate_image'" + "AND job.id=:id AND job.task_type='validate_image' AND image.id IS NOT NULL" ), {"tenant": actor.tenant_id, "id": job_id}, ) @@ -50,7 +68,8 @@ def dead_letters(self, connection, actor, page, size, laboratory_id=None): authorize(actor, Permission.ADMIN) params = {"tenant": actor.tenant_id} where = ( - "job.tenant_id=:tenant AND job.task_type='validate_image' " + "job.tenant_id=:tenant AND ((job.task_type='validate_image' AND image.id IS NOT NULL) " + "OR (job.task_type='report_export' AND report.id IS NOT NULL)) " "AND job.state IN ('failed','dead_letter')" ) if laboratory_id is not None: @@ -62,7 +81,10 @@ def dead_letters(self, connection, actor, page, size, laboratory_id=None): if lab is None: raise not_found() params["lab"] = laboratory_id - where += " AND image.laboratory_id=:lab" + where += ( + " AND (image.laboratory_id=:lab OR " + "JSON_CONTAINS(report.filters,JSON_QUOTE(:lab),'$.laboratory_ids'))" + ) total = connection.scalar(text(f"SELECT COUNT(*) FROM {FROM} WHERE {where}"), params) rows = connection.execute( text( @@ -81,10 +103,25 @@ def dead_letters(self, connection, actor, page, size, laboratory_id=None): def replay_source(self, connection, actor, job_id): # Same tenant -> owner -> upload -> image -> task order as claim/commit/recover. + report = ( + connection.execute( + text( + "SELECT * FROM task_runs WHERE tenant_id=:tenant AND id=:id " + "AND task_type='report_export'" + ), + {"tenant": actor.tenant_id, "id": job_id}, + ) + .mappings() + .first() + ) + if report is not None: + return reports.replay_source(connection, actor, report) return job_execution._load(connection, actor.tenant_id, job_id) def replay(self, connection, actor, task, reason, request_id): now = db_now(connection) + if task["task_type"] == "report_export": + reports.restore_failed(connection, actor, task["resource_id"], now) generation, sequence = task["replay_generation"] + 1, task["dispatch_sequence"] + 1 version = task["version"] + 1 message = validate_dispatch( @@ -93,7 +130,7 @@ def replay(self, connection, actor, task, reason, request_id): "tenant_id": actor.tenant_id, "trace_id": UUID(request_id).hex, "task_id": task["id"], - "task_type": "validate_image", + "task_type": task["task_type"], "resource_id": task["resource_id"], "replay_generation": generation, "dispatch_sequence": sequence, diff --git a/packages/persistence/migrations/versions/0002_report_object_version.py b/packages/persistence/migrations/versions/0002_report_object_version.py new file mode 100644 index 0000000..8777ba9 --- /dev/null +++ b/packages/persistence/migrations/versions/0002_report_object_version.py @@ -0,0 +1,27 @@ +"""Pin generated reports to an exact object version; preserve the released baseline.""" + +import os + +from alembic import op + +revision = "0002_report_object_version" +down_revision = "0001_initial" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + "ALTER TABLE report_exports ADD COLUMN object_version VARCHAR(200) NULL, " + "ADD COLUMN size_bytes BIGINT UNSIGNED NULL" + ) + + +def downgrade(): + schema = op.get_bind().engine.url.database + if ( + os.environ.get("APP_ENV") != "test" + or os.environ.get("LABSAFE_ALLOW_DESTRUCTIVE_DOWNGRADE") != schema + ): + raise RuntimeError("Destructive downgrade requires explicitly confirmed test schema") + op.execute("ALTER TABLE report_exports DROP COLUMN object_version, DROP COLUMN size_bytes") diff --git a/packages/persistence/remediation.py b/packages/persistence/remediation.py new file mode 100644 index 0000000..be5cb4f --- /dev/null +++ b/packages/persistence/remediation.py @@ -0,0 +1,621 @@ +"""Transactional remediation tasks and evidence.""" + +# SQL projections are kept aligned with the public projection contract. +# ruff: noqa: E501 + +from datetime import datetime, timezone +from uuid import uuid4 + +from sqlalchemy import text + +from packages.domain.security import Permission, ServiceError, authorize, not_found +from packages.domain.workflow import ( + Evidence, + Finding, + Image, + RemediationDecision, + Task, + accept_task, + cannot_remediate, + dispatch_finding, + reassign_task, + recheck_task, + reject_task, + submit_evidence, +) +from packages.persistence.dispatch import db_now +from packages.persistence.foundations import lab_filter, page_query, project +from packages.persistence.security import append_audit, load_principal +from packages.persistence.transaction import require_transaction +from packages.persistence.users import timestamp +from packages.shared.json_hash import canonical_json + + +def _row(connection, tenant, task_id, *, lock=False): + return ( + connection.execute( + text( + "SELECT t.id,t.created_at,t.updated_at,t.version,t.tenant_id,t.finding_id," + "t.laboratory_id,t.assignee_id,t.status,t.priority,t.due_at,t.description," + "t.latest_evidence_id,t.closed_at,f.version AS finding_version,f.status AS finding_status," + "f.item_id,f.run_id,f.rule_evaluation_id,f.superseded_at " + "FROM remediation_tasks t JOIN findings f ON f.tenant_id=t.tenant_id AND f.id=t.finding_id " + "WHERE t.tenant_id=:tenant AND t.id=:id" + (" FOR UPDATE" if lock else "") + ), + {"tenant": tenant, "id": task_id}, + ) + .mappings() + .first() + ) + + +def _finding_row(connection, tenant, finding_id, *, lock=False): + return ( + connection.execute( + text( + "SELECT f.id,f.tenant_id,r.laboratory_id,f.version,f.status,f.item_id,f.run_id," + "f.rule_evaluation_id,f.superseded_at FROM findings f " + "JOIN inference_runs r ON r.tenant_id=f.tenant_id AND r.item_id=f.item_id " + "AND r.id=f.run_id WHERE f.tenant_id=:tenant AND f.id=:id" + + (" FOR UPDATE" if lock else "") + ), + {"tenant": tenant, "id": finding_id}, + ) + .mappings() + .first() + ) + + +def _principal(connection, tenant, user_id): + return load_principal(connection, tenant, user_id) + + +def _task_domain(row): + return Task( + id=row["id"], + tenant_id=row["tenant_id"], + laboratory_id=row["laboratory_id"], + version=row["version"], + status=row["status"], + finding_id=row["finding_id"], + assignee_id=row["assignee_id"], + latest_evidence_id=row["latest_evidence_id"], + ) + + +def _finding_domain(row): + return Finding( + id=row.get("finding_id", row["id"]), + tenant_id=row["tenant_id"], + laboratory_id=row["laboratory_id"], + version=row["finding_version"], + status=row["finding_status"], + item_id=row["item_id"], + run_id=row["run_id"], + evaluation_id=row["rule_evaluation_id"], + superseded=row["superseded_at"] is not None, + evidence_ready=bool(row.get("latest_evidence_id")), + ) + + +def _project(row): + value = project(dict(row)) + for key in ("due_at", "closed_at"): + value[key] = timestamp(row[key]) if row[key] else None + value["is_overdue"] = ( + row["status"] not in {"closed", "cannot_remediate"} and row["due_at"] <= datetime.utcnow() + ) + value["allowed_actions"] = [] + value.pop("tenant_id", None) + value.pop("finding_version", None) + value.pop("finding_status", None) + value.pop("item_id", None) + value.pop("run_id", None) + value.pop("rule_evaluation_id", None) + value.pop("superseded_at", None) + return value + + +def _event(connection, tenant, event_type, task_id, version, payload, now): + event_id = str(uuid4()) + envelope = { + "schema_version": "1.1", + "tenant_id": tenant, + "trace_id": uuid4().hex, + "event_id": event_id, + "event_type": event_type, + "aggregate_type": "remediation_task", + "aggregate_id": task_id, + "occurred_at": timestamp(now), + "aggregate_version": version, + "payload": payload, + } + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) VALUES " + "(:id,:tenant,:type,'remediation_task',:task,:version,:payload,'pending',:now,:now,:now)" + ), + { + "id": event_id, + "tenant": tenant, + "type": event_type, + "task": task_id, + "version": version, + "payload": canonical_json(envelope), + "now": now, + }, + ) + return event_id + + +def _notify(connection, tenant, event_id, recipient, resource_id, kind): + connection.execute( + text( + "INSERT INTO notifications (id,tenant_id,event_id,recipient_id,type,resource_type,resource_id) " + "VALUES (:id,:tenant,:event,:recipient,:type,'remediation_task',:resource)" + ), + { + "id": str(uuid4()), + "tenant": tenant, + "event": event_id, + "recipient": recipient, + "type": kind, + "resource": resource_id, + }, + ) + + +def _commit_status( + connection, actor, row, decision: RemediationDecision, reason, request_id, action +): + now = db_now(connection) + task_version = row["version"] + 1 + changed = connection.execute( + text( + "UPDATE remediation_tasks SET status=:status,version=:version," + "closed_at=:closed,updated_at=:now WHERE tenant_id=:tenant AND id=:id AND version=:old" + ), + { + "status": str(decision.task_status), + "version": task_version, + "closed": now if str(decision.task_status) == "closed" else None, + "now": now, + "tenant": actor.tenant_id, + "id": row["id"], + "old": row["version"], + }, + ).rowcount + if changed != 1: + raise ServiceError("VERSION_CONFLICT", 409, "Remediation task changed during command") + if decision.finding_status != row["finding_status"]: + changed = connection.execute( + text( + "UPDATE findings SET status=:status,version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:id AND version=:version" + ), + { + "status": str(decision.finding_status), + "now": now, + "tenant": actor.tenant_id, + "id": row["finding_id"], + "version": row["finding_version"], + }, + ).rowcount + if changed != 1: + raise ServiceError("VERSION_CONFLICT", 409, "Finding changed during command") + event_type = { + "closed": "RemediationClosed", + "pending_recheck": "EvidenceSubmitted", + "pending_dispatch": "RemediationDispatched", + }.get(str(decision.task_status)) + event_id = ( + _event( + connection, + actor.tenant_id, + event_type, + row["id"], + task_version, + {"finding_id": row["finding_id"], "actor_id": actor.user_id}, + now, + ) + if event_type + else None + ) + if event_id and str(decision.task_status) == "pending_dispatch": + _notify( + connection, + actor.tenant_id, + event_id, + row["assignee_id"], + row["id"], + "remediation_assigned", + ) + connection.execute( + text( + "INSERT INTO review_actions (id,tenant_id,actor_id,resource_type,resource_id,action,reason,before_version,after_version) " + "VALUES (:id,:tenant,:actor,'remediation_task',:resource,:action,:reason,:before,:after)" + ), + { + "id": str(uuid4()), + "tenant": actor.tenant_id, + "actor": actor.user_id, + "resource": row["id"], + "action": action, + "reason": reason, + "before": row["version"], + "after": task_version, + }, + ) + append_audit( + connection, + principal=actor, + action=action, + resource_type="remediation_task", + resource_id=row["id"], + changes={"status": str(decision.task_status), "version": task_version}, + reason=reason, + request_id=request_id, + ) + return _project(_row(connection, actor.tenant_id, row["id"])) + + +def get_task(connection, actor, task_id): + row = _row(connection, actor.tenant_id, task_id) + if row is None: + raise not_found() + authorize(actor, Permission.READ, row["laboratory_id"]) + return _project(row) + + +def authorize_finding(connection, actor, finding_id): + row = _finding_row(connection, actor.tenant_id, finding_id) + if row is None: + raise not_found() + authorize(actor, Permission.DISPATCH, row["laboratory_id"]) + return row + + +def list_tasks(connection, actor, page, page_size, laboratory_id=None, status=None, overdue=None): + params = {"tenant": actor.tenant_id} + where = "t.tenant_id=:tenant" + lab_filter(actor, "t.laboratory_id", params, laboratory_id) + if status: + where += " AND t.status=:status" + params["status"] = status + if overdue is True: + where += " AND t.status NOT IN ('closed','cannot_remediate') AND t.due_at now + ) + + +def _current(report, lease): + return report is not None and report["status"] == "running" and _input(report) == lease.input + + +def _attempt(connection, task, status, code, now): + changed = connection.execute( + text( + "UPDATE task_attempts SET status=:status,error_code=:code,finished_at=:now," + "updated_at=:now,version=version+1 WHERE tenant_id=:tenant AND task_id=:id " + "AND replay_generation=:generation AND attempt=:attempt AND fencing_token=:token " + "AND lease_owner=:owner AND status='running'" + ), + { + "status": status, + "code": code, + "now": now, + "tenant": task["tenant_id"], + "id": task["id"], + "generation": task["replay_generation"], + "attempt": task["attempt"], + "token": task["fencing_token"], + "owner": task["lease_owner"], + }, + ).rowcount + if changed != 1: + raise RuntimeError("Missing current report attempt") + + +def _settle(connection, task, report, now, code, *, abandoned=False): + retry = report is not None and code in REPORT_RETRYABLE and task["attempt"] < 4 + state = "retry_wait" if retry else ("dead_letter" if code in REPORT_RETRYABLE else "failed") + _attempt(connection, task, "abandoned" if abandoned else "failed", code, now) + connection.execute( + text( + "UPDATE task_runs SET state=:state,last_error_code=:code,lease_owner=NULL," + "lease_until=NULL,heartbeat_at=NULL,fencing_token=fencing_token+1," + "available_at=:available,finished_at=:finished,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "state": state, + "code": code, + "now": now, + "tenant": task["tenant_id"], + "id": task["id"], + "available": now + retry_delay(task["attempt"]) if retry else now, + "finished": None if retry else now, + }, + ) + if report is not None: + connection.execute( + text( + "UPDATE report_exports SET status=:status,last_error_code=:code," + "version=version+1,updated_at=:now WHERE tenant_id=:tenant AND id=:id" + ), + { + "status": "queued" if retry else "failed", + "code": code, + "now": now, + "tenant": task["tenant_id"], + "id": report["id"], + }, + ) + return state + + +def claim(connection, message, owner): + validate_dispatch(message) + identifier(owner) + if message["task_type"] != "report_export": + raise ValueError("Expected report dispatch") + task, report = _load(connection, message["tenant_id"], message["task_id"]) + if task is None: + return None + for counter in ("replay_generation", "dispatch_sequence"): + if message[counter] > task[counter]: + raise ValueError("Future report dispatch") + if message[counter] < task[counter]: + return None + if message["resource_id"] != task["resource_id"] or message["payload"] != decoded( + task["payload"] + ): + raise ValueError("Report dispatch binding mismatch") + now = db_now(connection) + if ( + task["state"] not in {"ready", "retry_wait"} + or task["available_at"] > now + or task["attempt"] >= 4 + or report is None + or report["status"] != "queued" + or report["format"] != "csv" + ): + return None + if any( + report[name] is not None + for name in ("object_key", "object_version", "checksum", "size_bytes", "expires_at") + ): + raise ValueError("Queued report already has an artifact") + source = replace(_input(report), version=report["version"] + 1) + attempt, token = task["attempt"] + 1, task["fencing_token"] + 1 + params = { + "tenant": task["tenant_id"], + "id": task["id"], + "attempt": attempt, + "token": token, + "owner": owner, + "now": now, + "until": now + timedelta(seconds=60), + } + connection.execute( + text( + "UPDATE task_runs SET state='leased',attempt=:attempt,fencing_token=:token," + "lease_owner=:owner,lease_until=:until,heartbeat_at=:now,started_at=COALESCE(started_at,:now)," + "finished_at=NULL,last_error_code=NULL,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + params, + ) + connection.execute( + text( + "INSERT INTO task_attempts (id,tenant_id,task_id,replay_generation,attempt," + "fencing_token,lease_owner,started_at,status,created_at,updated_at) VALUES " + "(:attempt_id,:tenant,:id,:generation,:attempt,:token,:owner,:now,'running',:now,:now)" + ), + {**params, "attempt_id": str(uuid4()), "generation": task["replay_generation"]}, + ) + connection.execute( + text( + "UPDATE report_exports SET status='running',last_error_code=NULL," + "version=version+1,updated_at=:now WHERE tenant_id=:tenant AND id=:id" + ), + {"tenant": task["tenant_id"], "id": report["id"], "now": now}, + ) + return ReportLease( + task["tenant_id"], task["id"], owner, token, task["replay_generation"], attempt, source + ) + + +def heartbeat(connection, lease): + task, report = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or not _current(report, lease): + return False + connection.execute( + text( + "UPDATE task_runs SET lease_until=:until,heartbeat_at=:now,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "until": now + timedelta(seconds=60), + "now": now, + "tenant": lease.tenant_id, + "id": lease.task_id, + }, + ) + return True + + +def fail(connection, lease, code): + if code not in REPORT_ERRORS: + raise ValueError("Unknown report error") + task, report = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or not _current(report, lease): + return False + _settle(connection, task, report, now, code) + return True + + +def commit_result(connection, lease, artifact): + validate_artifact(lease.input, artifact) + task, report = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or not _current(report, lease): + raise ReportLeaseLost() + connection.execute( + text( + "UPDATE report_exports SET status='ready',object_key=:key,checksum=:checksum," + "object_version=:object_version,size_bytes=:size,expires_at=:expiry," + "last_error_code=NULL,version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "key": artifact.key, + "checksum": artifact.checksum, + "object_version": artifact.object_version, + "size": artifact.size_bytes, + "expiry": now + timedelta(hours=24), + "now": now, + "tenant": lease.tenant_id, + "id": report["id"], + }, + ) + _attempt(connection, task, "succeeded", None, now) + append_audit( + connection, + principal=SimpleNamespace(tenant_id=lease.tenant_id, user_id=None), + action="report.export_ready", + resource_type="report_export", + resource_id=report["id"], + changes={ + "status": "ready", + "checksum": artifact.checksum, + "size_bytes": artifact.size_bytes, + }, + reason="Generate CSV from fixed report snapshot", + request_id=lease.task_id, + ) + connection.execute( + text( + "UPDATE task_runs SET state='succeeded',lease_owner=NULL,lease_until=NULL," + "heartbeat_at=NULL,last_error_code=NULL,finished_at=:now," + "updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + {"now": now, "tenant": lease.tenant_id, "id": lease.task_id}, + ) + return True + + +def expired_candidates(connection, limit=100): + require_transaction(connection) + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("Batch limit must be 1..100") + return [ + dict(row) + for row in connection.execute( + text( + "SELECT tenant_id,id,fencing_token,replay_generation FROM task_runs " + "WHERE task_type='report_export' AND state='leased' " + "AND lease_until<=UTC_TIMESTAMP(3) " + "ORDER BY lease_until,id LIMIT :limit" + ), + {"limit": limit}, + ).mappings() + ] + + +def recover(connection, candidate): + task, report = _load(connection, candidate["tenant_id"], candidate["id"]) + now = db_now(connection) + if ( + task is None + or task["state"] != "leased" + or task["lease_until"] is None + or task["lease_until"] > now + or task["fencing_token"] != candidate["fencing_token"] + or task["replay_generation"] != candidate["replay_generation"] + ): + return False + if report is None or report["status"] != "running" or report["format"] != "csv": + _settle(connection, task, None, now, "LEASE_LOST", abandoned=True) + else: + _settle(connection, task, report, now, "STAGE_TIMEOUT", abandoned=True) + return True diff --git a/packages/persistence/reports.py b/packages/persistence/reports.py new file mode 100644 index 0000000..1d9ebf5 --- /dev/null +++ b/packages/persistence/reports.py @@ -0,0 +1,299 @@ +"""Freeze authorized report fields and enqueue work in a caller-owned transaction.""" + +from uuid import UUID, uuid4 + +from sqlalchemy import text + +from packages.domain.reports import COLUMNS, MAX_FINDINGS, validate_request +from packages.domain.security import Permission, ServiceError, authorize, not_found +from packages.persistence.dispatch import db_now, decoded, validate_dispatch +from packages.persistence.security import append_audit +from packages.persistence.transaction import require_transaction +from packages.persistence.users import timestamp +from packages.shared.json_hash import canonical_json + +# Every join is tenant-scoped; only the current run/revision/evaluation contributes findings. +SOURCE = ( + "inspection_items ii JOIN inspections i ON i.tenant_id=ii.tenant_id " + "AND i.id=ii.inspection_id AND i.laboratory_id=ii.laboratory_id " + "LEFT JOIN inference_runs r ON r.tenant_id=ii.tenant_id " + "AND r.item_id=ii.id AND r.id=ii.current_run_id " + "LEFT JOIN findings f ON f.tenant_id=ii.tenant_id AND f.item_id=ii.id " + "AND f.run_id=ii.current_run_id AND f.fact_revision_id=ii.current_fact_revision_id " + "AND f.rule_evaluation_id=ii.current_evaluation_id AND f.superseded_at IS NULL " + "LEFT JOIN remediation_tasks rt ON rt.tenant_id=f.tenant_id AND rt.finding_id=f.id" +) + + +def authorize_labs(connection, actor, labs, permission): + if not labs: + raise ServiceError("STATE_CONFLICT", 409, "Report scope is missing") + params = {"tenant": actor.tenant_id, **{f"lab{i}": lab for i, lab in enumerate(labs)}} + names = ",".join(f":lab{i}" for i in range(len(labs))) + present = ( + connection.execute( + text(f"SELECT id FROM laboratories WHERE tenant_id=:tenant AND id IN ({names})"), + params, + ) + .scalars() + .all() + ) + if set(present) != set(labs): + raise not_found() + for lab in labs: + authorize(actor, permission, lab) + + +def _source(filters, tenant, start, end): + params = { + "tenant": tenant, + "start": start, + "end": end, + **{f"lab{i}": lab for i, lab in enumerate(filters["laboratory_ids"])}, + } + labs = ",".join(f":lab{i}" for i in range(len(filters["laboratory_ids"]))) + where = f"ii.tenant_id=:tenant AND ii.laboratory_id IN ({labs})" + # Dates refer to inspection creation, inclusive in UTC. Zero-finding items remain rows. + where += " AND i.created_at>=:start AND i.created_at<=:end" + for name, column in (("severity", "severity"), ("finding_status", "status")): + values = filters[name] + if values: + binds = ",".join(f":{name}{i}" for i in range(len(values))) + params.update({f"{name}{i}": value for i, value in enumerate(values)}) + where += f" AND (f.{column} IN ({binds}) OR f.id IS NULL)" + return where, params + + +def _snapshot_rows(connection, filters, tenant, start, end, now): + where, params = _source(filters, tenant, start, end) + count = connection.scalar(text(f"SELECT COUNT(f.id) FROM {SOURCE} WHERE {where}"), params) + if count > MAX_FINDINGS: + raise ServiceError( + "VALIDATION_ERROR", 422, "Export exceeds 10000 findings; narrow the range" + ) + rows = connection.execute( + text( + "SELECT i.id AS inspection_id,ii.id AS item_id,ii.laboratory_id,ii.location_id," + "ii.current_run_id AS run_id,ii.current_fact_revision_id AS fact_revision_id," + "r.rule_bundle_id,f.id AS finding_id,f.rule_id,f.severity,f.status,f.explanation," + f"rt.status AS task_status,ii.review_outcome FROM {SOURCE} WHERE {where} " + "ORDER BY ii.laboratory_id,i.id,ii.id,f.id" + ), + params, + ).mappings() + return [{**dict(row), "snapshot_at": timestamp(now)} for row in rows], count + + +def get(connection, actor, export_id, *, permission=Permission.READ, current=False): + require_transaction(connection) + row = ( + connection.execute( + text( + "SELECT e.id,e.created_at,e.updated_at,e.version,e.format,e.status,e.snapshot_at," + "e.filters,e.expires_at,e.last_error_code,t.id AS job_id FROM report_exports e " + "JOIN task_runs t ON t.tenant_id=e.tenant_id AND t.task_type='report_export' " + "AND t.resource_id=e.id AND t.logical_key=CONCAT('report_export:',e.id) " + "WHERE e.tenant_id=:tenant AND e.id=:id" + (" FOR SHARE" if current else "") + ), + {"tenant": actor.tenant_id, "id": export_id}, + ) + .mappings() + .first() + ) + if row is None: + raise not_found() + authorize_labs(connection, actor, decoded(row["filters"])["laboratory_ids"], permission) + return { + "id": row["id"], + "created_at": timestamp(row["created_at"]), + "updated_at": timestamp(row["updated_at"]), + "version": row["version"], + "format": row["format"], + "status": row["status"], + "snapshot_at": timestamp(row["snapshot_at"]), + "expires_at": timestamp(row["expires_at"]) if row["expires_at"] else None, + "job_id": row["job_id"], + "error_code": row["last_error_code"], + } + + +def download_row(connection, actor, export_id): + """Load private download evidence under the current tenant transaction.""" + require_transaction(connection) + return ( + connection.execute( + text( + "SELECT id,tenant_id,requested_by,format,status,filters,expires_at," + "object_key,checksum,object_version,size_bytes FROM report_exports " + "WHERE tenant_id=:tenant AND id=:id FOR SHARE" + ), + {"tenant": actor.tenant_id, "id": export_id}, + ) + .mappings() + .first() + ) + + +def _outbox(connection, tenant, kind, aggregate, resource, payload, now): + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) " + "VALUES (:id,:tenant,:kind,:aggregate,:resource,1,:payload,'pending',:now,:now,:now)" + ), + { + "id": payload.get("event_id", str(uuid4())), + "tenant": tenant, + "kind": kind, + "aggregate": aggregate, + "resource": resource, + "payload": canonical_json(payload), + "now": now, + }, + ) + + +def create(connection, actor, body, request_id): + require_transaction(connection) + filters, start, end = validate_request(body) + authorize_labs(connection, actor, filters["laboratory_ids"], Permission.EXPORT) + now = db_now(connection) + rows, count = _snapshot_rows(connection, filters, actor.tenant_id, start, end, now) + export_id, task_id = str(uuid4()), str(uuid4()) + snapshot = { + "schema_version": "1", + "export_id": export_id, + "tenant_id": actor.tenant_id, + "laboratory_ids": filters["laboratory_ids"], + "snapshot_at": timestamp(now), + "columns": list(COLUMNS), + "rows": rows, + } + connection.execute( + text( + "INSERT INTO report_exports (id,tenant_id,requested_by,format,filters,snapshot_at," + "snapshot,status,created_at,updated_at) " + "VALUES (:id,:tenant,:user,:format,:filters,:now,:snapshot,'queued',:now,:now)" + ), + { + "id": export_id, + "tenant": actor.tenant_id, + "user": actor.user_id, + "format": body["format"], + "filters": canonical_json(filters), + "now": now, + "snapshot": canonical_json(snapshot), + }, + ) + payload = {"export_id": export_id} + connection.execute( + text( + "INSERT INTO task_runs (id,tenant_id,task_type,resource_id,logical_key,payload," + "state,available_at,last_dispatched_at,created_at,updated_at) " + "VALUES (:id,:tenant,'report_export',:resource,:key,:payload," + "'ready',:now,:now,:now,:now)" + ), + { + "id": task_id, + "tenant": actor.tenant_id, + "resource": export_id, + "key": f"report_export:{export_id}", + "payload": canonical_json(payload), + "now": now, + }, + ) + dispatch = validate_dispatch( + { + "schema_version": "1.1", + "tenant_id": actor.tenant_id, + "trace_id": UUID(request_id).hex, + "task_id": task_id, + "task_type": "report_export", + "resource_id": export_id, + "replay_generation": 0, + "dispatch_sequence": 1, + "created_at": timestamp(now), + "payload": payload, + } + ) + _outbox(connection, actor.tenant_id, "TaskDispatch", "task_run", task_id, dispatch, now) + envelope = { + "schema_version": "1.1", + "tenant_id": actor.tenant_id, + "trace_id": UUID(request_id).hex, + "event_id": str(uuid4()), + "event_type": "ReportRequested", + "aggregate_type": "report_export", + "aggregate_id": export_id, + "occurred_at": timestamp(now), + "aggregate_version": 1, + "payload": payload, + } + _outbox( + connection, actor.tenant_id, "ReportRequested", "report_export", export_id, envelope, now + ) + append_audit( + connection, + principal=actor, + action="report.export_requested", + resource_type="report_export", + resource_id=export_id, + changes={"format": body["format"], "finding_count": count}, + reason="Report export requested", + request_id=request_id, + ) + return get(connection, actor, export_id) + + +def replay_source(connection, actor, locator): + """Lock the report before its task; reject missing or mismatched immutable input.""" + report = ( + connection.execute( + text("SELECT * FROM report_exports WHERE tenant_id=:tenant AND id=:id FOR UPDATE"), + {"tenant": actor.tenant_id, "id": locator["resource_id"]}, + ) + .mappings() + .first() + ) + task = ( + connection.execute( + text("SELECT * FROM task_runs WHERE tenant_id=:tenant AND id=:id FOR UPDATE"), + {"tenant": actor.tenant_id, "id": locator["id"]}, + ) + .mappings() + .first() + ) + if report is None or task is None: + return task, None + filters = decoded(report["filters"]) + authorize_labs(connection, actor, filters["laboratory_ids"], Permission.EXPORT) + snapshot = decoded(report["snapshot"]) + valid = ( + report["status"] == "failed" + and task["task_type"] == "report_export" + and task["resource_id"] == report["id"] + and task["logical_key"] == f"report_export:{report['id']}" + and decoded(task["payload"]) == {"export_id": report["id"]} + and isinstance(snapshot, dict) + and snapshot.get("schema_version") == "1" + and snapshot.get("export_id") == report["id"] + and snapshot.get("tenant_id") == actor.tenant_id + and snapshot.get("laboratory_ids") == filters["laboratory_ids"] + and snapshot.get("columns") == list(COLUMNS) + and isinstance(snapshot.get("rows"), list) + and report["version"] < 2147483647 + ) + return task, report if valid else None + + +def restore_failed(connection, actor, export_id, now): + changed = connection.execute( + text( + "UPDATE report_exports SET status='queued',last_error_code=NULL," + "version=version+1,updated_at=:now WHERE tenant_id=:tenant AND id=:id " + "AND status='failed'" + ), + {"tenant": actor.tenant_id, "id": export_id, "now": now}, + ).rowcount + if changed != 1: + raise ServiceError("STATE_CONFLICT", 409, "Report export is not replayable") diff --git a/packages/persistence/schema.py b/packages/persistence/schema.py index 643139c..c11bdb4 100644 --- a/packages/persistence/schema.py +++ b/packages/persistence/schema.py @@ -13,7 +13,17 @@ def model(): - return json.loads(MODEL_PATH.read_text(encoding="utf-8")) + value = json.loads(MODEL_PATH.read_text(encoding="utf-8")) + # Additive head migration; never rewrite the released 0001 snapshots. + for name, sql_type in (("object_version", "VARCHAR(200)"), ("size_bytes", "BIGINT UNSIGNED")): + value["tables"]["report_exports"]["columns"][name] = { + "type": sql_type, + "nullable": True, + "default_sql": None, + "description": "", + "generated_sql": None, + } + return value def expression_tokens(value): @@ -228,7 +238,7 @@ def rows(query): ) compare(f"{table}.{name}.enforced", checks[table][name]["ENFORCED"], "YES") head = connection.scalar(text("SELECT version_num FROM alembic_version")) - compare("migration_head", head, "0001_initial") + compare("migration_head", head, "0002_report_object_version") compare("session.time_zone", connection.scalar(text("SELECT @@session.time_zone")), "+00:00") actual_modes = set(connection.scalar(text("SELECT @@session.sql_mode")).split(",")) required_modes = set(STRICT_SQL_MODE.split(",")) diff --git a/packages/storage/s3.py b/packages/storage/s3.py index 7d8cd64..5003421 100644 --- a/packages/storage/s3.py +++ b/packages/storage/s3.py @@ -15,6 +15,19 @@ from botocore.exceptions import BotoCoreError, ClientError from packages.domain.image_download import DOWNLOAD_SECONDS, validate_reference +from packages.domain.report_download import ( + REPORT_DOWNLOAD_SECONDS, + ReportDownload, +) +from packages.domain.report_download import ( + validate_reference as validate_report_reference, +) +from packages.domain.report_execution import ( + MAX_REPORT_BYTES, + REPORT_MIME, + artifact_for, + report_key, +) from packages.domain.security import ServiceError from packages.domain.uploads import GRANT_SECONDS, MAX_BYTES, MIME_TYPES, digest, staging_key @@ -248,6 +261,56 @@ def presign_image_get(self, reference): except (BotoCoreError, ClientError, ValueError, KeyError, TypeError): raise dependency() from None + def presign_report_get(self, reference: ReportDownload): + """Local SigV4 for one CSV report and its exact stored VersionId.""" + validate_report_reference(reference) + try: + url = self.signing.generate_presigned_url( + "get_object", + Params={ + "Bucket": BUCKET, + "Key": reference.key, + "VersionId": reference.object_version, + }, + ExpiresIn=REPORT_DOWNLOAD_SECONDS, + HttpMethod="GET", + ) + parts, origin = urlsplit(url), urlsplit(self.settings.public_origin) + query = parse_qs(parts.query, strict_parsing=True) + if ( + parts.scheme != origin.scheme + or parts.netloc != origin.netloc + or parts.path != f"/{BUCKET}/{reference.key}" + or parts.fragment + or len(url) > 4096 + or set(query) + != { + "versionId", + "X-Amz-Algorithm", + "X-Amz-Credential", + "X-Amz-Date", + "X-Amz-Expires", + "X-Amz-SignedHeaders", + "X-Amz-Signature", + } + or any(len(values) != 1 for values in query.values()) + or query.get("versionId") != [reference.object_version] + or query.get("X-Amz-Algorithm") != ["AWS4-HMAC-SHA256"] + or query.get("X-Amz-Expires") != [str(REPORT_DOWNLOAD_SECONDS)] + or query.get("X-Amz-SignedHeaders") != ["host"] + ): + raise ValueError + started = datetime.strptime(query["X-Amz-Date"][0], "%Y%m%dT%H%M%SZ") + credential = f"{self.settings.access_key}/{started:%Y%m%d}/us-east-1/s3/aws4_request" + if ( + query.get("X-Amz-Credential") != [credential] + or re.fullmatch(r"[a-f0-9]{64}", query["X-Amz-Signature"][0]) is None + ): + raise ValueError + return DownloadGrant(url, started + timedelta(seconds=REPORT_DOWNLOAD_SECONDS)) + except (BotoCoreError, ClientError, ValueError, KeyError, TypeError): + raise dependency() from None + @staticmethod def _failure(error): if isinstance(error, ClientError) and error.response.get("Error", {}).get("Code") in { @@ -454,3 +517,20 @@ def put_analysis(self, key, payload): return self.result_version(result) except (BotoCoreError, ClientError, OSError) as error: raise self._failure(error) from None + + def put_report(self, source, payload): + """Upload deterministic CSV bytes; register only an explicit VersionId.""" + if not isinstance(payload, bytes) or not 1 <= len(payload) <= MAX_REPORT_BYTES: + raise ServiceError("INTERNAL_ERROR", 500, "Report encoding exceeds limit") + checksum = hashlib.sha256(payload).hexdigest() + key = report_key(source, checksum) + version = self.reuse_version(key, checksum, REPORT_MIME, MAX_REPORT_BYTES) + if version is None: + try: + result = self.internal.put_object( + Bucket=BUCKET, Key=key, Body=payload, ContentType=REPORT_MIME + ) + version = self.result_version(result) + except (BotoCoreError, ClientError, OSError) as error: + raise self._failure(error) from None + return artifact_for(source, payload, version) diff --git a/tests/business/test_identity_application.py b/tests/business/test_identity_application.py index c538688..7e38389 100644 --- a/tests/business/test_identity_application.py +++ b/tests/business/test_identity_application.py @@ -37,7 +37,9 @@ def test_non_deadlock_database_errors_are_not_retried(errno): assert command.call_count == 1 -@pytest.mark.parametrize("revision", ["0001_initial", "unknown", None]) +@pytest.mark.parametrize( + "revision", ["0002_report_object_version", "0001_initial", "unknown", None] +) def test_readiness_checks_revision_and_releases_connection_before_redis(revision): connected = False connection = Mock() @@ -59,7 +61,7 @@ def ready(): engine.connect.side_effect = connect limits.ready.side_effect = ready app = IdentityApplication(engine, Mock(), limits, "test") - if revision == "0001_initial": + if revision == "0002_report_object_version": app.readiness() limits.ready.assert_called_once_with() else: diff --git a/tests/business/test_remediation_boundary.py b/tests/business/test_remediation_boundary.py new file mode 100644 index 0000000..25f9ff7 --- /dev/null +++ b/tests/business/test_remediation_boundary.py @@ -0,0 +1,89 @@ +"""Strict remediation HTTP contracts without database dependencies.""" + +from unittest.mock import Mock +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient + +from apps.api.app.main import create_app +from tests.business.test_identity_boundary import ORIGIN +from tests.helpers import configured + +ID = str(uuid4()) +HEADERS = {"Origin": ORIGIN, "Idempotency-Key": "remediation-key", "X-CSRF-Token": "s" * 43} +DISPATCH = { + "expected_version": 1, + "assignee_id": ID, + "due_at": "2030-01-01T00:00:00Z", + "priority": "high", + "description": "Contain the confirmed risk", +} + + +def client(service): + with configured(): + app = create_app(identity=Mock(), public_origin=ORIGIN) + app.state.remediation = service + return TestClient(app, base_url=ORIGIN) + + +def test_remediation_writes_keep_strict_body_and_security_boundary(): + service = Mock() + service.write.return_value = (201, {"data": {}, "request_id": ID}) + with client(service) as http: + response = http.post( + f"/api/v1/findings/{ID}/remediation-tasks", json=DISPATCH, headers=HEADERS + ) + assert response.status_code == 201 + assert response.headers["cache-control"] == "no-store" + assert ( + http.post( + f"/api/v1/findings/{ID}/remediation-tasks", + json={**DISPATCH, "tenant_id": ID}, + headers=HEADERS, + ).status_code + == 422 + ) + assert ( + http.post( + f"/api/v1/findings/{ID}/remediation-tasks", + json={**DISPATCH, "due_at": "2030-01-01T00:00:00"}, + headers=HEADERS, + ).status_code + == 422 + ) + assert service.write.call_args.args[0] == "dispatch" + assert service.write.call_args.args[1] is None + assert service.write.call_args.args[5]["due_at"] == DISPATCH["due_at"] + + +@pytest.mark.parametrize( + "query", + ["overdue=maybe", "page=1&page=2", "unknown=1", "laboratory_id=bad"], +) +def test_remediation_list_rejects_ambiguous_or_unknown_queries(query): + service = Mock() + with client(service) as http: + response = http.get(f"/api/v1/remediation-tasks?{query}") + assert response.status_code == 422 + service.read.assert_not_called() + + +def test_remediation_routes_expose_contract_operation_ids(): + with client(Mock()) as http: + operations = { + route.operation_id for route in http.app.routes if hasattr(route, "operation_id") + } + assert { + "dispatchRemediation", + "listRemediations", + "getRemediation", + "acceptRemediation", + "submitevidenceRemediation", + "recheckRemediation", + "rejectRemediation", + "cannotremediateRemediation", + "reassignRemediation", + "listEvidence", + } <= operations diff --git a/tests/business/test_report_boundary.py b/tests/business/test_report_boundary.py new file mode 100644 index 0000000..344f25a --- /dev/null +++ b/tests/business/test_report_boundary.py @@ -0,0 +1,63 @@ +"""Strict report/export HTTP contracts without database dependencies.""" + +from unittest.mock import Mock +from uuid import uuid4 + +from fastapi.testclient import TestClient + +from apps.api.app.main import create_app +from tests.business.test_identity_boundary import ORIGIN +from tests.helpers import configured + +ID = str(uuid4()) +HEADERS = {"Origin": ORIGIN, "Idempotency-Key": "report-key", "X-CSRF-Token": "s" * 43} +BODY = { + "format": "csv", + "filters": { + "laboratory_ids": [ID], + "from": "2030-01-01T00:00:00Z", + "to": "2030-01-31T00:00:00Z", + "severity": ["high"], + "finding_status": ["confirmed"], + }, +} + + +def client(service): + with configured(): + app = create_app(identity=Mock(), public_origin=ORIGIN) + app.state.reports = service + return TestClient(app, base_url=ORIGIN) + + +def test_create_export_is_strict_and_async(): + service = Mock() + service.create.return_value = (202, {"data": {}, "request_id": ID}) + with client(service) as http: + response = http.post("/api/v1/reports/exports", json=BODY, headers=HEADERS) + assert response.status_code == 202 + assert ( + http.post( + "/api/v1/reports/exports", json={**BODY, "tenant_id": ID}, headers=HEADERS + ).status_code + == 422 + ) + assert ( + http.post( + "/api/v1/reports/exports", + json={**BODY, "filters": {**BODY["filters"], "to": "2030-01-31T00:00:00"}}, + headers=HEADERS, + ).status_code + == 422 + ) + assert service.create.call_args.args[3]["filters"]["laboratory_ids"] == [ID] + + +def test_report_queries_reject_query_parameters_and_expose_operations(): + service = Mock() + with client(service) as http: + assert http.get(f"/api/v1/reports/exports/{ID}?page=1").status_code == 422 + operations = { + route.operation_id for route in http.app.routes if hasattr(route, "operation_id") + } + assert {"createExport", "getExport", "replayJob"} <= operations diff --git a/tests/business/test_report_download.py b/tests/business/test_report_download.py new file mode 100644 index 0000000..3812486 --- /dev/null +++ b/tests/business/test_report_download.py @@ -0,0 +1,86 @@ +"""Report download contract and exact-version signing without object network access.""" + +from datetime import datetime, timedelta +from unittest.mock import Mock, patch +from urllib.parse import parse_qs, urlsplit +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient + +from apps.api.app.main import create_app +from packages.domain.report_download import ReportDownload, validate_reference +from packages.storage.s3 import BUCKET, S3Storage +from tests.business.test_foundation_boundary import ORIGIN +from tests.business.test_s3_storage import settings +from tests.helpers import configured + +TENANT, EXPORT, LAB = str(uuid4()), str(uuid4()), str(uuid4()) +CHECKSUM = "a" * 64 + + +def reference(**changes): + value = ReportDownload( + TENANT, + EXPORT, + (LAB,), + CHECKSUM, + f"tenant/{TENANT}/lab/{LAB}/reports/{EXPORT}/{CHECKSUM}.csv", + "report-v1", + 123, + "text/csv; charset=utf-8", + ) + return value.__class__( + *(changes.get(name, getattr(value, name)) for name in value.__dataclass_fields__) + ) + + +def test_report_reference_binds_scope_checksum_key_version_and_size(): + validate_reference(reference()) + for changes in ( + {"key": "staging/report"}, + {"checksum": "bad"}, + {"object_version": "null"}, + {"size_bytes": 0}, + {"mime_type": "application/pdf"}, + ): + with pytest.raises(ValueError): + validate_reference(reference(**changes)) + + +def test_report_signer_uses_exact_version_and_60_second_ttl_without_network(): + storage = S3Storage(settings()) + instant = datetime(2026, 10, 7, 1, 2, 3) + try: + with ( + patch("botocore.auth.get_current_datetime", return_value=instant), + patch( + "botocore.httpsession.URLLib3Session.send", side_effect=AssertionError("network") + ), + ): + grant = storage.presign_report_get(reference(object_version="version/+?=&:v1")) + parts, query = urlsplit(grant.url), parse_qs(urlsplit(grant.url).query) + assert parts.path == f"/{BUCKET}/{reference().key}" + assert query["versionId"] == ["version/+?=&:v1"] + assert query["X-Amz-Expires"] == ["60"] + assert grant.expires_at == instant + timedelta(seconds=60) + finally: + storage.close() + + +def test_report_download_route_rejects_query_and_returns_grant(): + with configured(): + app = create_app(identity=Mock(), public_origin=ORIGIN, storage=Mock()) + service = Mock() + service.storage = Mock() + service.download.return_value = { + "data": {"url": "https://labsafe.test/signed", "expires_at": "time"}, + "request_id": str(uuid4()), + } + app.state.reports = service + with TestClient(app, base_url=ORIGIN) as http: + response = http.get(f"/api/v1/reports/exports/{EXPORT}/download") + assert response.status_code == 200 + assert response.headers["referrer-policy"] == "no-referrer" + assert http.get(f"/api/v1/reports/exports/{EXPORT}/download?tenant_id=x").status_code == 422 + service.download.assert_called_once() diff --git a/tests/business/test_report_execution.py b/tests/business/test_report_execution.py new file mode 100644 index 0000000..dc98e24 --- /dev/null +++ b/tests/business/test_report_execution.py @@ -0,0 +1,335 @@ +"""Frozen CSV encoding, pinned S3 artifacts and bounded worker orchestration.""" + +import csv +import hashlib +import io +import json +import multiprocessing +import time +from dataclasses import replace +from pathlib import Path +from threading import Event, Timer +from types import SimpleNamespace +from unittest.mock import Mock, patch +from uuid import uuid4 + +import pytest +from botocore.response import StreamingBody +from botocore.stub import Stubber + +from apps.worker.report_export import BoundedReportPrepare +from packages.application.job_execution import LeaseHeartbeat +from packages.application.report_execution import ReportExecution +from packages.domain.job_execution import LeaseLost +from packages.domain.report_execution import ( + REPORT_ERRORS, + REPORT_MIME, + REPORT_RETRYABLE, + ReportInput, + ReportInvalid, + artifact_for, + csv_bytes, + report_key, + validate_artifact, +) +from packages.domain.reports import COLUMNS +from packages.domain.security import ServiceError +from packages.shared.json_hash import canonical_json +from packages.storage.s3 import BUCKET, S3Settings, S3Storage +from tests.business.test_s3_storage import settings + + +def source(text="说明,含逗号\n第二行"): + tenant, export, lab = str(uuid4()), str(uuid4()), str(uuid4()) + instant = "2026-10-06T00:00:00.000Z" + row = dict.fromkeys(COLUMNS) + row.update(laboratory_id=lab, snapshot_at=instant, explanation=text) + snapshot = { + "schema_version": "1", + "tenant_id": tenant, + "export_id": export, + "laboratory_ids": [lab], + "snapshot_at": instant, + "columns": list(COLUMNS), + "rows": [row], + } + return ReportInput( + export, + tenant, + str(uuid4()), + "csv", + canonical_json({"laboratory_ids": [lab]}), + instant, + canonical_json(snapshot), + 2, + ) + + +@pytest.mark.parametrize("prefix", ["=", "+", "-", "@", "\t", "\r"]) +def test_report_csv_formula_prefixes_are_escaped(prefix): + value = source(prefix + '中文,"value"\nnext') + encoded = csv_bytes(value) + assert encoded.startswith(b"\xef\xbb\xbf") + rows = list(csv.reader(io.StringIO(encoded.decode("utf-8-sig"), newline=""))) + assert rows[0] == list(COLUMNS) + assert rows[1][COLUMNS.index("explanation")] == "'" + prefix + '中文,"value"\nnext' + assert rows[1][0] == "" + assert csv_bytes(value) == encoded + + +def test_report_csv_quotes_unicode_newlines_and_header_only_snapshot(): + value = source() + rows = list(csv.reader(io.StringIO(csv_bytes(value).decode("utf-8-sig"), newline=""))) + assert rows[1][COLUMNS.index("explanation")] == "说明,含逗号\n第二行" + frozen = json.loads(value.snapshot) + frozen["rows"] = [] + empty = replace(value, snapshot=canonical_json(frozen)) + assert csv_bytes(empty).decode("utf-8-sig") == ",".join(COLUMNS) + "\r\n" + + +@pytest.mark.parametrize( + "fault", + [ + "tenant", + "export", + "labs", + "columns", + "time", + "row_scope", + "row_value", + "extra", + "json", + "pdf", + ], +) +def test_report_invalid_snapshot_cannot_generate_bytes(fault): + value = source() + frozen = json.loads(value.snapshot) + if fault in {"tenant", "export"}: + frozen[fault + "_id"] = str(uuid4()) + elif fault == "labs": + frozen["laboratory_ids"] = [] + elif fault == "columns": + frozen["columns"].reverse() + elif fault == "time": + frozen["snapshot_at"] = "2026-01-01T00:00:00Z" + elif fault == "row_scope": + frozen["rows"][0]["laboratory_id"] = str(uuid4()) + elif fault == "row_value": + frozen["rows"][0]["explanation"] = {"private": "value"} + elif fault == "extra": + frozen["user_email"] = "private" + value = replace( + value, + snapshot="{" if fault == "json" else canonical_json(frozen), + format="pdf" if fault == "pdf" else "csv", + ) + with pytest.raises(ReportInvalid): + csv_bytes(value) + + +def test_report_csv_output_bound_and_cancellation(monkeypatch): + monkeypatch.setattr("packages.domain.report_execution.MAX_REPORT_BYTES", 10) + with pytest.raises(ReportInvalid): + csv_bytes(source()) + with pytest.raises(LeaseLost): + csv_bytes(source(), Mock(side_effect=LeaseLost())) + + +def test_report_key_uses_smallest_lab_and_artifact_requires_exact_version(): + value = source() + frozen, filters = json.loads(value.snapshot), json.loads(value.filters) + labs = sorted([*filters["laboratory_ids"], str(uuid4())]) + frozen["laboratory_ids"] = filters["laboratory_ids"] = labs + value = replace(value, snapshot=canonical_json(frozen), filters=canonical_json(filters)) + payload = csv_bytes(value) + result = artifact_for(value, payload, "pinned-version") + assert ( + result.key == f"tenant/{value.tenant_id}/lab/{labs[0]}/reports/{value.export_id}/" + f"{hashlib.sha256(payload).hexdigest()}.csv" + ) + for bad in ( + replace(result, object_version="null"), + replace(result, key=result.key + ".pdf"), + replace(result, size_bytes=0), + replace(result, object_version="bad\nversion"), + ): + with pytest.raises((ReportInvalid, ServiceError)): + validate_artifact(value, bad) + + +def test_report_error_policy_matches_contract(): + contract = json.loads( + (Path(__file__).resolve().parents[2] / "contracts/error-codes.json").read_text() + ) + assert all(contract[code]["retryable"] == (code in REPORT_RETRYABLE) for code in REPORT_ERRORS) + + +@pytest.mark.parametrize("version", ["new-version", "null", ""]) +def test_report_put_records_only_explicit_object_version(version): + value = source() + payload = csv_bytes(value) + key = report_key(value, hashlib.sha256(payload).hexdigest()) + storage = S3Storage(settings()) + try: + with Stubber(storage.internal) as stub: + stub.add_client_error( + "head_object", + service_error_code="NoSuchKey", + http_status_code=404, + expected_params={"Bucket": BUCKET, "Key": key}, + ) + stub.add_response( + "put_object", + {"VersionId": version}, + {"Bucket": BUCKET, "Key": key, "Body": payload, "ContentType": REPORT_MIME}, + ) + if version == "new-version": + assert storage.put_report(value, payload) == artifact_for(value, payload, version) + else: + with pytest.raises(ServiceError) as failure: + storage.put_report(value, payload) + assert failure.value.code == "DEPENDENCY_UNAVAILABLE" + stub.assert_no_pending_responses() + finally: + storage.close() + + +@pytest.mark.parametrize("conflict", [False, True]) +def test_report_reuses_existing_version_only_after_hashing_its_bytes(conflict): + value = source() + payload = csv_bytes(value) + key = report_key(value, hashlib.sha256(payload).hexdigest()) + stored = b"x" * len(payload) if conflict else payload + storage = S3Storage(settings()) + stream = StreamingBody(io.BytesIO(stored), len(stored)) + metadata = {"VersionId": "existing", "ContentLength": len(stored), "ContentType": REPORT_MIME} + try: + with Stubber(storage.internal) as stub: + stub.add_response("head_object", metadata, {"Bucket": BUCKET, "Key": key}) + stub.add_response( + "get_object", + {**metadata, "Body": stream}, + {"Bucket": BUCKET, "Key": key, "VersionId": "existing"}, + ) + if conflict: + with pytest.raises(ServiceError) as failure: + storage.put_report(value, payload) + assert failure.value.code == "INTERNAL_ERROR" + else: + assert storage.put_report(value, payload).object_version == "existing" + stub.assert_no_pending_responses() + finally: + storage.close() + assert stream._raw_stream.closed + + +def execution(monkeypatch): + monkeypatch.setenv("APP_ENV", "test") + service = ReportExecution(Mock()) + service.claim = Mock(return_value=SimpleNamespace(task_id="report", input=source())) + service.heartbeat = Mock(return_value=True) + service.commit = Mock() + service.fail = Mock() + return service + + +@pytest.mark.parametrize( + "error,code", + [ + (ReportInvalid(), "SCHEMA_MISMATCH"), + (ServiceError("DEPENDENCY_UNAVAILABLE", 503, "private"), "DEPENDENCY_UNAVAILABLE"), + (RuntimeError("private"), "INTERNAL_ERROR"), + ], +) +def test_report_execution_classifies_failure(monkeypatch, error, code): + service = execution(monkeypatch) + with pytest.raises(type(error)): + service.execute({}, Mock(side_effect=error)) + service.fail.assert_called_once_with(service.claim.return_value, code) + service.commit.assert_not_called() + + +def test_report_execution_duplicate_does_not_prepare(monkeypatch): + service = execution(monkeypatch) + service.claim.return_value = None + prepare = Mock() + assert service.execute({}, prepare) is False + prepare.assert_not_called() + + +def test_report_heartbeat_failure_prevents_failure_or_success_writes(monkeypatch): + service = execution(monkeypatch) + service.heartbeat.return_value = False + monkeypatch.setattr(LeaseHeartbeat, "interval", 0.01) + + def prepare(source, cancelled): + assert cancelled.wait(2) + return artifact_for(source, csv_bytes(source), "orphan") + + with pytest.raises(LeaseLost): + service.execute({}, prepare) + service.commit.assert_not_called() + service.fail.assert_not_called() + + +def stalled_report(*args): + time.sleep(60) + + +@pytest.mark.parametrize("cancel", [False, True]) +def test_report_child_deadline_and_cancellation_reap_process(monkeypatch, cancel): + monkeypatch.setattr("apps.worker.report_export.prepare_child", stalled_report) + monkeypatch.setattr("apps.worker.report_export.REPORT_SECONDS", 5 if cancel else 0.2) + cancelled = Event() + timer = Timer(0.1, cancelled.set) if cancel else None + before = {p.pid for p in multiprocessing.active_children()} + try: + if timer: + timer.start() + with pytest.raises(LeaseLost if cancel else ServiceError) as error: + BoundedReportPrepare(settings())(source(), cancelled) + if not cancel: + assert error.value.code == "STAGE_TIMEOUT" + finally: + if timer: + timer.cancel() + timer.join() + assert {p.pid for p in multiprocessing.active_children()} == before + + +def test_report_worker_is_opt_in_and_separate_queue(monkeypatch): + from apps.worker.app.main import create_celery_app + from apps.worker.dispatch import send_dispatch + from apps.worker.run import main + + monkeypatch.setenv("APP_ENV", "test") + monkeypatch.setenv("PUBLIC_ORIGIN", "https://labsafe.test") + for name in ("IMAGE_VALIDATION", "INFERENCE", "RULE_EVALUATION", "REPORT_EXPORT"): + monkeypatch.setenv(f"WORKER_{name}_ENABLED", "0") + app = create_celery_app() + try: + assert "labsafe.tasks.dispatch" not in app.tasks + finally: + app.close() + monkeypatch.setenv("WORKER_REPORT_EXPORT_ENABLED", "1") + with ( + patch.object(S3Settings, "from_environment", return_value=settings()), + patch("apps.worker.report_export.consume_report") as consume, + ): + app = create_celery_app() + try: + app.tasks["labsafe.tasks.dispatch"].run({"task_type": "report_export"}) + consume.assert_called_once() + finally: + app.close() + fake = Mock() + with patch("apps.worker.dispatch.create_celery_app", return_value=fake): + send_dispatch(str(uuid4()), {"task_type": "report_export"}) + assert fake.send_task.call_args.kwargs["queue"] == "q.reports" + with ( + patch("apps.worker.run.create_celery_app", return_value=fake), + patch("sys.argv", ["worker"]), + ): + main() + assert "--queues=q.reports" in fake.worker_main.call_args.args[0] diff --git a/tests/business/test_report_guards.py b/tests/business/test_report_guards.py new file mode 100644 index 0000000..1419bd4 --- /dev/null +++ b/tests/business/test_report_guards.py @@ -0,0 +1,67 @@ +"""Range limits and persisted report dispatch protocol.""" + +from copy import deepcopy +from uuid import uuid4 + +import pytest + +from packages.domain.reports import validate_request +from packages.domain.security import ServiceError +from packages.persistence.dispatch import validate_dispatch +from tests.business.test_dispatch import message +from tests.business.test_report_boundary import BODY + + +def test_export_31_days_and_100_labs_are_accepted_in_utc(): + request = deepcopy(BODY) + request["filters"].update( + { + "laboratory_ids": [str(uuid4()) for _ in range(100)], + "from": "2030-01-01T08:00:00+08:00", + "to": "2030-02-01T00:00:00Z", + } + ) + normalized, start, end = validate_request(request) + assert normalized["from"] == "2030-01-01T00:00:00.000Z" + assert (end - start).days == 31 + + +@pytest.mark.parametrize( + "field,value", + [ + ("laboratory_ids", []), + ("laboratory_ids", [str(uuid4()) for _ in range(101)]), + ("laboratory_ids", ["bad"]), + ("laboratory_ids", BODY["filters"]["laboratory_ids"] * 2), + ("from", "2030-01-01"), + ("to", "2029-12-31T00:00:00Z"), + ("to", "2030-02-01T00:00:00.001Z"), + ("severity", ["wrong"]), + ("finding_status", ["confirmed", "confirmed"]), + ], +) +def test_export_invalid_ranges_fail_before_persistence(field, value): + request = deepcopy(BODY) + request["filters"][field] = value + with pytest.raises(ServiceError) as error: + validate_request(request) + assert error.value.status == 422 + + +@pytest.mark.parametrize("fault", [None, "missing", "extra", "mismatched", "invalid"]) +def test_report_dispatch_requires_exact_export_binding(fault): + value = message() + value.update(task_type="report_export", payload={"export_id": value["resource_id"]}) + if fault == "missing": + value["payload"] = {} + elif fault == "extra": + value["payload"]["object_key"] = "not-allowed" + elif fault == "mismatched": + value["payload"]["export_id"] = str(uuid4()) + elif fault == "invalid": + value["payload"]["export_id"] = value["resource_id"] = "bad" + if fault: + with pytest.raises((ValueError, ServiceError)): + validate_dispatch(value) + else: + assert validate_dispatch(value) == value diff --git a/tests/persistence/conftest.py b/tests/persistence/conftest.py index d225c9f..20c2f1e 100644 --- a/tests/persistence/conftest.py +++ b/tests/persistence/conftest.py @@ -5,6 +5,7 @@ import socket import subprocess import time +from uuid import uuid4 import pytest from alembic import command @@ -19,6 +20,7 @@ from packages.persistence.database import database_engine from packages.persistence.schema import verify_schema from packages.persistence.security import SessionService +from tests.persistence.factories import insert from tests.persistence.test_security_mysql import create_tenant @@ -40,7 +42,37 @@ def database(): with engine.begin() as connection: assert connection.scalar(text("SELECT id FROM user_sentinel")) == 7 connection.execute(text("DROP TABLE user_sentinel")) + command.upgrade(config, "0001_initial") + # Verify the additive report migration against a populated old schema. + tenant_id, user_id, export_id = str(uuid4()), str(uuid4()), str(uuid4()) + with engine.begin() as connection: + insert(connection, "tenants", id=tenant_id) + insert(connection, "users", id=user_id, tenant_id=tenant_id) + insert( + connection, + "report_exports", + id=export_id, + tenant_id=tenant_id, + requested_by=user_id, + snapshot='{"preserve":"frozen input"}', + ) + original = dict( + connection.execute(text("SELECT * FROM report_exports WHERE id=:id"), {"id": export_id}) + .mappings() + .one() + ) command.upgrade(config, "head") + with engine.begin() as connection: + upgraded = dict( + connection.execute(text("SELECT * FROM report_exports WHERE id=:id"), {"id": export_id}) + .mappings() + .one() + ) + assert {key: upgraded[key] for key in original} == original + assert upgraded["object_version"] is upgraded["size_bytes"] is None + connection.execute(text("DELETE FROM report_exports WHERE id=:id"), {"id": export_id}) + connection.execute(text("DELETE FROM users WHERE id=:id"), {"id": user_id}) + connection.execute(text("DELETE FROM tenants WHERE id=:id"), {"id": tenant_id}) with engine.connect() as connection: report = verify_schema(connection) print(json.dumps(report, indent=2)) diff --git a/tests/persistence/test_remediation_mysql.py b/tests/persistence/test_remediation_mysql.py new file mode 100644 index 0000000..93e1d66 --- /dev/null +++ b/tests/persistence/test_remediation_mysql.py @@ -0,0 +1,287 @@ +"""Real MySQL acceptance for the remediation task lifecycle.""" + +import json +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text + +from tests.persistence.factories import insert +from tests.persistence.test_identity_mysql import ORIGIN, login +from tests.persistence.test_item_queries_mysql import inspection +from tests.persistence.test_roles_mysql import command, create_user, role_write + + +def post(http, session, path, body, *, key=None): + return http.post( + f"/api/v1{path}", + json=body, + headers={ + "Origin": ORIGIN, + "X-CSRF-Token": session["data"]["csrf_token"], + "Idempotency-Key": key or str(uuid4()), + }, + ) + + +def finding_case(http, database, tenant, session): + value, lab, _, _ = inspection(http, session, count=1) + item_id = value["item_ids"][0] + with database.begin() as connection: + dictionary = insert( + connection, "dictionary_versions", tenant_id=tenant["tenant_id"], status="published" + ) + model = insert( + connection, + "model_versions", + tenant_id=tenant["tenant_id"], + dictionary_version_id=dictionary, + status="published", + ) + rules = insert(connection, "rule_bundles", tenant_id=tenant["tenant_id"]) + run = insert( + connection, + "inference_runs", + tenant_id=tenant["tenant_id"], + item_id=item_id, + laboratory_id=lab["id"], + model_bundle_id=model, + dictionary_version_id=dictionary, + rule_bundle_id=rules, + status="needs_review", + stage="done", + result=json.dumps({"detections": []}), + ) + fact = insert( + connection, + "fact_revisions", + tenant_id=tenant["tenant_id"], + item_id=item_id, + run_id=run, + entities="[]", + relations="[]", + dates="[]", + ) + evaluation = insert( + connection, + "rule_evaluations", + tenant_id=tenant["tenant_id"], + item_id=item_id, + run_id=run, + fact_revision_id=fact, + rule_bundle_id=rules, + status="completed", + ) + finding = insert( + connection, + "findings", + tenant_id=tenant["tenant_id"], + item_id=item_id, + run_id=run, + fact_revision_id=fact, + rule_evaluation_id=evaluation, + status="confirmed", + evidence=json.dumps({"raw_facts": {"entities": [], "relations": []}}), + ) + return finding, lab["id"] + + +@pytest.mark.usefixtures("database") +def test_dispatch_idempotency_and_atomic_write_set(identity, database): + tenant, _, app = identity + with TestClient(app, base_url=ORIGIN) as http: + admin = login(http, tenant).json() + user = create_user(http, admin, "remediator") + value, lab = finding_case(http, database, tenant, admin) + role_result = role_write(http, admin, user["id"], command(role="remediator", lab=lab)) + assert role_result.status_code == 200 + due_at = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + body = { + "expected_version": 1, + "assignee_id": user["id"], + "due_at": due_at, + "priority": "high", + "description": "Contain the confirmed risk", + } + key = "remediation-dispatch-1" + first = post(http, admin, f"/findings/{value}/remediation-tasks", body, key=key) + replay = post(http, admin, f"/findings/{value}/remediation-tasks", body, key=key) + assert first.status_code == replay.status_code == 201, first.text + assert first.content == replay.content + task_id = first.json()["data"]["id"] + with database.connect() as connection: + state = ( + connection.execute( + text( + "SELECT t.status,t.version,f.status AS finding_status," + "(SELECT COUNT(*) FROM outbox_events WHERE aggregate_id=:task) AS events," + "(SELECT COUNT(*) FROM notifications " + "WHERE resource_id=:task) AS notifications," + "(SELECT COUNT(*) FROM audit_events WHERE resource_id=:task) AS audits " + "FROM remediation_tasks t JOIN findings f ON f.id=t.finding_id " + "WHERE t.id=:task" + ), + {"task": task_id}, + ) + .mappings() + .one() + ) + assert dict(state) == { + "status": "pending_dispatch", + "version": 1, + "finding_status": "dispatched", + "events": 1, + "notifications": 1, + "audits": 1, + } + replacement = create_user(http, admin, "replacement") + role_result = role_write( + http, admin, replacement["id"], command(role="remediator", lab=lab) + ) + assert role_result.status_code == 200 + reassigned = post( + http, + admin, + f"/remediation-tasks/{task_id}/reassign", + { + "expected_version": 1, + "assignee_id": replacement["id"], + "due_at": due_at, + "reason": "Move to the replacement remediator", + }, + ) + assert reassigned.status_code == 200, reassigned.text + assert reassigned.json()["data"]["assignee_id"] == replacement["id"] + with database.connect() as connection: + notification = connection.execute( + text( + "SELECT recipient_id FROM notifications " + "WHERE resource_id=:task ORDER BY created_at DESC LIMIT 1" + ), + {"task": task_id}, + ).scalar_one() + assert notification == replacement["id"] + + +@pytest.mark.usefixtures("database") +def test_remediation_accept_submit_and_recheck_are_fenced(identity, database): + tenant, _, app = identity + with TestClient(app, base_url=ORIGIN) as http: + admin = login(http, tenant).json() + user = create_user(http, admin, "remediator") + finding, lab = finding_case(http, database, tenant, admin) + role_result = role_write(http, admin, user["id"], command(role="remediator", lab=lab)) + assert role_result.status_code == 200 + due_at = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + dispatched = post( + http, + admin, + f"/findings/{finding}/remediation-tasks", + { + "expected_version": 1, + "assignee_id": user["id"], + "due_at": due_at, + "priority": "medium", + "description": "Investigate", + }, + ) + assert dispatched.status_code == 201, dispatched.text + task_id = dispatched.json()["data"]["id"] + remediator = login(http, tenant, username="remediator").json() + accepted = post( + http, + remediator, + f"/remediation-tasks/{task_id}/accept", + { + "expected_version": 1, + "reason": "Accepted for remediation", + }, + ) + assert accepted.status_code == 200, accepted.text + assert accepted.json()["data"]["status"] == "in_progress" + + with database.begin() as connection: + upload = insert( + connection, + "uploads", + tenant_id=tenant["tenant_id"], + laboratory_id=lab, + remediation_task_id=task_id, + requested_by=user["id"], + object_key=f"staging/{uuid4()}", + expected_sha256="b" * 64, + mime_type="image/png", + size_bytes=100, + ) + image = insert( + connection, + "asset_images", + tenant_id=tenant["tenant_id"], + laboratory_id=lab, + remediation_task_id=task_id, + upload_id=upload, + status="ready", + original_key=f"original/{uuid4()}", + original_object_version="original-v1", + original_sha256="b" * 64, + analysis_key=f"analysis/{uuid4()}", + analysis_object_version="analysis-v1", + analysis_sha256="a" * 64, + width=100, + height=100, + mime_type="image/png", + ) + submitted = post( + http, + remediator, + f"/remediation-tasks/{task_id}/submit-evidence", + { + "expected_version": 2, + "image_ids": [image], + "description": "Containment evidence", + }, + ) + assert submitted.status_code == 200, submitted.text + evidence = submitted.json()["data"] + assert evidence["image_ids"] == [image] + stale = post( + http, + remediator, + f"/remediation-tasks/{task_id}/submit-evidence", + { + "expected_version": 2, + "image_ids": [image], + "description": "Stale retry", + }, + ) + assert stale.status_code == 409 + assert stale.json()["error"]["code"] == "VERSION_CONFLICT" + + admin = login(http, tenant).json() + checked = post( + http, + admin, + f"/remediation-tasks/{task_id}/recheck", + { + "expected_version": 3, + "evidence_id": evidence["id"], + "reason": "Evidence verified", + }, + ) + assert checked.status_code == 200, checked.text + assert checked.json()["data"]["status"] == "closed" + with database.connect() as connection: + state = ( + connection.execute( + text( + "SELECT t.status,t.version,f.status AS finding_status " + "FROM remediation_tasks t JOIN findings f ON f.id=:finding WHERE t.id=:task" + ), + {"finding": finding, "task": task_id}, + ) + .mappings() + .one() + ) + assert dict(state) == {"status": "closed", "version": 4, "finding_status": "closed"} diff --git a/tests/persistence/test_report_download_mysql.py b/tests/persistence/test_report_download_mysql.py new file mode 100644 index 0000000..cfc8002 --- /dev/null +++ b/tests/persistence/test_report_download_mysql.py @@ -0,0 +1,72 @@ +"""Report download authorization and exact object evidence on disposable MySQL.""" + +from datetime import timedelta +from unittest.mock import Mock + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text + +from packages.persistence.security import utc_now +from packages.storage.s3 import DownloadGrant +from tests.persistence.test_identity_mysql import ORIGIN, headers, login +from tests.persistence.test_report_export_mysql import body +from tests.persistence.test_roles_mysql import lab_fixture + + +@pytest.fixture +def download_case(identity, database): + tenant, service, app = identity + lab = lab_fixture(database, tenant) + with TestClient(app, base_url=ORIGIN) as http: + session = login(http, tenant).json() + yield tenant, service, http, session, body([lab]) + + +def post_export(http, session, request): + return http.post("/api/v1/reports/exports", json=request, headers=headers(session)) + + +def test_ready_report_download_signs_exact_registered_object(download_case, database): + tenant, _, http, session, request = download_case + created = post_export(http, session, request) + assert created.status_code == 202, created.text + export_id = created.json()["data"]["id"] + with database.begin() as connection: + row = ( + connection.execute( + text("SELECT tenant_id,filters FROM report_exports WHERE id=:id"), {"id": export_id} + ) + .mappings() + .one() + ) + labs = row["filters"]["laboratory_ids"] if isinstance(row["filters"], dict) else None + if labs is None: + import json + + labs = json.loads(row["filters"])["laboratory_ids"] + checksum = "a" * 64 + key = f"tenant/{row['tenant_id']}/lab/{labs[0]}/reports/{export_id}/{checksum}.csv" + connection.execute( + text( + "UPDATE report_exports SET status='ready',object_key=:key,checksum=:checksum," + "object_version='exact-v1',size_bytes=123,expires_at=:expires WHERE id=:id" + ), + { + "id": export_id, + "key": key, + "checksum": checksum, + "expires": utc_now() + timedelta(hours=1), + }, + ) + service = http.app.state.reports + service.storage = Mock() + service.storage.presign_report_get.return_value = DownloadGrant( + "https://labsafe.test/signed", utc_now() + timedelta(seconds=60) + ) + response = http.get(f"/api/v1/reports/exports/{export_id}/download") + assert response.status_code == 200, response.text + reference = service.storage.presign_report_get.call_args.args[0] + assert reference.key == key and reference.object_version == "exact-v1" + assert response.headers["cache-control"] == "no-store" + assert response.headers["referrer-policy"] == "no-referrer" diff --git a/tests/persistence/test_report_execution_mysql.py b/tests/persistence/test_report_execution_mysql.py new file mode 100644 index 0000000..2016337 --- /dev/null +++ b/tests/persistence/test_report_execution_mysql.py @@ -0,0 +1,475 @@ +"""CSV execution races, fences and transaction failures on disposable MySQL.""" + +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import timedelta +from threading import Barrier +from unittest.mock import Mock +from uuid import uuid4 + +import pytest +from sqlalchemy import event, text + +from packages.application.dispatch import DispatchPublisher, transaction +from packages.application.report_execution import ReportExecution +from packages.domain.job_execution import LeaseLost +from packages.domain.job_replay import require_replay +from packages.domain.report_execution import ReportInvalid, artifact_for, csv_bytes +from packages.persistence import dispatch +from packages.persistence import report_execution as execution +from packages.persistence.dispatch import decoded +from packages.persistence.jobs import JobRepository +from tests.persistence.factories import Graph +from tests.persistence.test_report_export_mysql import admin, body, create + + +@pytest.fixture +def case(database): + with database.begin() as connection: + graph = Graph(connection) + for item, run, fact, evaluation in ( + (graph.item, graph.run, graph.fact, graph.evaluation), + (graph.other_item, graph.other_run, graph.other_fact, graph.other_evaluation), + ): + connection.execute( + text( + "UPDATE inspection_items SET current_run_id=:run," + "current_fact_revision_id=:fact," + "current_evaluation_id=:evaluation WHERE id=:id" + ), + {"run": run, "fact": fact, "evaluation": evaluation, "id": item}, + ) + result = create(connection, graph) + graph.export, graph.job = result["id"], result["job_id"] + graph.message = decoded( + connection.scalar( + text( + "SELECT payload FROM outbox_events WHERE aggregate_id=:job " + "AND event_type='TaskDispatch'" + ), + {"job": graph.job}, + ) + ) + yield graph + # Only retire this fixture's notices/tasks; keep its business graph in the disposable DB. + with database.begin() as connection: + for name in ("task_attempts", "outbox_events", "task_runs"): + connection.execute( + text(f"DELETE FROM {name} WHERE tenant_id=:tenant"), {"tenant": graph.tenant} + ) + + +def row(database, table, key): + with database.connect() as connection: + return dict( + connection.execute(text(f"SELECT * FROM {table} WHERE id=:id"), {"id": key}) + .mappings() + .one() + ) + + +def update(database, statement, **params): + with database.begin() as connection: + connection.execute(text(statement), params) + + +def claim(database, case, message=None): + with transaction(database) as connection: + return execution.claim(connection, message or case.message, str(uuid4())) + + +def records(database, case): + with database.connect() as connection: + return { + name: [ + dict(r) + for r in connection.execute( + text(f"SELECT * FROM {name} WHERE tenant_id=:tenant ORDER BY id"), + {"tenant": case.tenant}, + ).mappings() + ] + for name in ( + "report_exports", + "task_runs", + "task_attempts", + "outbox_events", + "audit_events", + ) + } + + +def artifact(lease): + return artifact_for(lease.input, csv_bytes(lease.input), "test-exact-version") + + +def test_report_execution_success_pins_artifact_and_duplicate_is_noop(database, case): + lease = claim(database, case) + assert lease.attempt == lease.token == 1 + assert row(database, "report_exports", case.export)["status"] == "running" + assert claim(database, case) is None + with transaction(database) as connection: + assert execution.heartbeat(connection, lease) + assert execution.commit_result(connection, lease, artifact(lease)) + result, job = row(database, "report_exports", case.export), row(database, "task_runs", case.job) + assert result["status"] == "ready" and job["state"] == "succeeded" + assert result["object_version"] == "test-exact-version" + assert result["checksum"] == artifact(lease).checksum + assert result["object_key"] == artifact(lease).key + assert result["size_bytes"] == len(csv_bytes(lease.input)) + assert result["expires_at"] - job["finished_at"] == timedelta(hours=24) + saved = records(database, case) + assert saved["task_attempts"][0]["status"] == "succeeded" + ready_audit = [r for r in saved["audit_events"] if r["action"] == "report.export_ready"] + assert len(ready_audit) == 1 and ready_audit[0]["actor_id"] is None + assert {r["event_type"] for r in saved["outbox_events"]} == {"TaskDispatch", "ReportRequested"} + assert claim(database, case) is None + with transaction(database) as connection: + assert not execution.fail(connection, lease, "INTERNAL_ERROR") + with pytest.raises(LeaseLost): + execution.commit_result(connection, lease, artifact(lease)) + assert records(database, case) == saved + + +def test_report_execution_concurrent_claim_has_one_attempt(database, case): + barrier = Barrier(2) + + def run(_): + barrier.wait(timeout=5) + return claim(database, case) + + with ThreadPoolExecutor(max_workers=2) as pool: + leases = list(pool.map(run, range(2))) + assert sum(lease is not None for lease in leases) == 1 + assert len(records(database, case)["task_attempts"]) == 1 + + +@pytest.mark.parametrize("counter", ["replay_generation", "dispatch_sequence"]) +def test_report_execution_old_and_future_notifications(database, case, counter): + update(database, f"UPDATE task_runs SET {counter}={counter}+1 WHERE id=:id", id=case.job) + before = records(database, case) + assert claim(database, case) is None + with pytest.raises(ValueError, match="Future"): + claim(database, case, {**case.message, counter: case.message[counter] + 2}) + assert records(database, case) == before + assert claim(database, case, {**case.message, counter: case.message[counter] + 1}) is not None + + +@pytest.mark.parametrize("fault", ["tenant", "resource", "pdf", "not_due"]) +def test_report_execution_claim_guards(database, case, fault): + message = dict(case.message) + if fault == "tenant": + message["tenant_id"] = str(uuid4()) + elif fault == "resource": + message["resource_id"] = str(uuid4()) + message["payload"] = {"export_id": message["resource_id"]} + elif fault == "pdf": + update(database, "UPDATE report_exports SET format='pdf' WHERE id=:id", id=case.export) + else: + update( + database, + "UPDATE task_runs SET available_at=UTC_TIMESTAMP(3)+INTERVAL 1 DAY WHERE id=:id", + id=case.job, + ) + before = records(database, case) + if fault == "resource": + with pytest.raises(ValueError): + claim(database, case, message) + else: + assert claim(database, case, message) is None + assert records(database, case) == before + + +@pytest.mark.parametrize( + "fault", + ["owner", "token", "generation", "attempt", "snapshot", "filters", "version", "expired"], +) +def test_report_execution_changed_ownership_or_input_rejects_all_writes(database, case, fault): + lease = claim(database, case) + submitted = lease + if fault in {"owner", "token", "generation", "attempt"}: + submitted = replace( + lease, **{fault: str(uuid4()) if fault == "owner" else getattr(lease, fault) + 1} + ) + elif fault == "snapshot": + update( + database, + "UPDATE report_exports SET snapshot=JSON_SET(snapshot,'$.rows',JSON_ARRAY()) " + "WHERE id=:id", + id=case.export, + ) + elif fault == "filters": + update( + database, + "UPDATE report_exports SET filters=JSON_SET(filters,'$.severity',JSON_ARRAY('high')) " + "WHERE id=:id", + id=case.export, + ) + elif fault == "version": + update(database, "UPDATE report_exports SET version=version+1 WHERE id=:id", id=case.export) + else: + update( + database, + "UPDATE task_runs SET lease_until=UTC_TIMESTAMP(3)-INTERVAL 1 SECOND WHERE id=:id", + id=case.job, + ) + before = records(database, case) + with transaction(database) as connection: + assert not execution.heartbeat(connection, submitted) + assert not execution.fail(connection, submitted, "INTERNAL_ERROR") + with pytest.raises(LeaseLost): + execution.commit_result(connection, submitted, artifact(lease)) + assert records(database, case) == before + + +@pytest.mark.parametrize( + "code,expected", + [ + ("DEPENDENCY_UNAVAILABLE", "retry_wait"), + ("STAGE_TIMEOUT", "retry_wait"), + ("SCHEMA_MISMATCH", "failed"), + ("INTERNAL_ERROR", "failed"), + ], +) +def test_report_execution_failure_converges_report_task_and_attempt(database, case, code, expected): + lease = claim(database, case) + with transaction(database) as connection: + assert execution.fail(connection, lease, code) + result, job = row(database, "report_exports", case.export), row(database, "task_runs", case.job) + assert job["state"] == expected and job["fencing_token"] > lease.token + assert result["status"] == ("queued" if expected == "retry_wait" else "failed") + assert result["last_error_code"] == job["last_error_code"] == code + assert result["object_key"] is result["object_version"] is None + attempt = records(database, case)["task_attempts"][0] + assert attempt["status"] == "failed" and attempt["error_code"] == code + if expected == "retry_wait": + delay = (job["available_at"] - attempt["finished_at"]).total_seconds() + assert 5 <= delay <= 6 + + +def test_report_execution_four_expirations_dead_letter_and_old_lease_is_fenced(database, case): + old = None + for attempt in range(1, 5): + update( + database, "UPDATE task_runs SET available_at=UTC_TIMESTAMP(3) WHERE id=:id", id=case.job + ) + lease = claim(database, case) + assert lease.attempt == attempt + old = old or lease + update( + database, + "UPDATE task_runs SET lease_until=UTC_TIMESTAMP(3)-INTERVAL 1 SECOND WHERE id=:id", + id=case.job, + ) + with transaction(database) as connection: + candidates = execution.expired_candidates(connection) + candidate = next(r for r in candidates if r["id"] == case.job) + assert execution.recover(connection, candidate) + assert not execution.recover(connection, candidate) + job = row(database, "task_runs", case.job) + assert job["state"] == ("retry_wait" if attempt < 4 else "dead_letter") + assert row(database, "report_exports", case.export)["status"] == ( + "queued" if attempt < 4 else "failed" + ) + before = records(database, case) + assert len(before["task_attempts"]) == 4 + assert all(r["status"] == "abandoned" for r in before["task_attempts"]) + with transaction(database) as connection: + assert not execution.fail(connection, old, "INTERNAL_ERROR") + with pytest.raises(LeaseLost): + execution.commit_result(connection, old, artifact(old)) + assert records(database, case) == before + + +@pytest.mark.parametrize( + "phase,statement", + [ + ("claim", "UPDATE task_runs"), + ("claim", "INSERT INTO task_attempts"), + ("claim", "UPDATE report_exports"), + ("commit", "UPDATE report_exports"), + ("commit", "UPDATE task_attempts"), + ("commit", "INSERT INTO audit_events"), + ("commit", "UPDATE task_runs"), + ("fail", "UPDATE task_attempts"), + ("fail", "UPDATE task_runs"), + ("fail", "UPDATE report_exports"), + ], +) +def test_report_execution_write_failures_roll_back_complete_write_set( + database, case, phase, statement +): + lease = None if phase == "claim" else claim(database, case) + before = records(database, case) + + def fail(connection, cursor, sql, params, context, many): + if sql.startswith(statement): + raise RuntimeError("injected report execution write failure") + + event.listen(database, "before_cursor_execute", fail) + try: + with pytest.raises(RuntimeError, match="injected"): + with transaction(database) as connection: + if phase == "claim": + execution.claim(connection, case.message, str(uuid4())) + elif phase == "commit": + execution.commit_result(connection, lease, artifact(lease)) + else: + execution.fail(connection, lease, "DEPENDENCY_UNAVAILABLE") + finally: + event.remove(database, "before_cursor_execute", fail) + assert records(database, case) == before + + +def test_report_execution_prepare_uses_frozen_data_outside_database_locks(database, case): + update( + database, + "UPDATE findings SET explanation='changed after snapshot' WHERE id=:id", + id=case.finding, + ) + service = ReportExecution(database) + + def prepare(source, cancelled): + # An independent connection can immediately lock both rows while external I/O runs. + with transaction(database) as connection: + connection.execute( + text("SELECT id FROM report_exports WHERE id=:id FOR UPDATE NOWAIT"), + {"id": case.export}, + ) + connection.execute( + text("SELECT id FROM task_runs WHERE id=:id FOR UPDATE NOWAIT"), {"id": case.job} + ) + payload = csv_bytes(source) + assert b"changed after snapshot" not in payload + return artifact_for(source, payload, "prepared-outside-db") + + assert service.execute(case.message, prepare) + assert row(database, "report_exports", case.export)["object_version"] == "prepared-outside-db" + + +def test_report_execution_invalid_snapshot_becomes_permanent_failure(database, case): + update(database, "UPDATE report_exports SET snapshot='{}' WHERE id=:id", id=case.export) + with pytest.raises(ReportInvalid): + ReportExecution(database).execute(case.message, lambda source, cancelled: csv_bytes(source)) + assert row(database, "report_exports", case.export)["last_error_code"] == "SCHEMA_MISMATCH" + assert row(database, "task_runs", case.job)["state"] == "failed" + + +def test_report_execution_publish_and_redispatch_gate_csv_only(database, case): + with database.begin() as connection: + pdf = create(connection, case, {**body([case.lab]), "format": "pdf"}) + connection.execute( + text("UPDATE outbox_events SET available_at='1990-01-01' WHERE tenant_id=:tenant"), + {"tenant": case.tenant}, + ) + transport = Mock() + DispatchPublisher(database, transport).publish_batch(limit=1) + assert not any( + call.args[1].get("tenant_id") == case.tenant for call in transport.publish.call_args_list + ) + transport.reset_mock() + assert DispatchPublisher(database, transport, report_enabled=True).publish_batch(limit=1) == 1 + assert transport.publish.call_args.args[1]["task_id"] == case.job + with transaction(database) as connection: + connection.execute( + text( + "UPDATE task_runs SET last_dispatched_at=NULL,available_at='1990-01-01' " + "WHERE tenant_id=:tenant" + ), + {"tenant": case.tenant}, + ) + dispatch.redispatch(connection, report_enabled=False) + assert ( + connection.scalar( + text("SELECT dispatch_sequence FROM task_runs WHERE id=:id"), {"id": case.job} + ) + == 1 + ) + dispatch.redispatch(connection, report_enabled=True) + assert row(database, "task_runs", case.job)["dispatch_sequence"] == 2 + assert row(database, "task_runs", pdf["job_id"])["dispatch_sequence"] == 1 + assert row(database, "report_exports", pdf["id"])["status"] == "queued" + assert claim(database, case) is None # The old notification was superseded. + assert claim(database, case, {**case.message, "dispatch_sequence": 2}) is not None + + +def test_report_execution_expired_publication_returns_to_pending(database, case): + update( + database, + "UPDATE outbox_events SET state='leased',lease_owner=:owner,lease_until='1990-01-01' " + "WHERE tenant_id=:tenant AND event_type='TaskDispatch'", + owner=str(uuid4()), + tenant=case.tenant, + ) + with transaction(database) as connection: + dispatch.recover_publications(connection, report_enabled=False) + assert ( + connection.scalar( + text( + "SELECT state FROM outbox_events WHERE aggregate_id=:id " + "AND event_type='TaskDispatch'" + ), + {"id": case.job}, + ) + == "leased" + ) + dispatch.recover_publications(connection, report_enabled=True) + assert ( + connection.scalar( + text( + "SELECT state FROM outbox_events WHERE aggregate_id=:id " + "AND event_type='TaskDispatch'" + ), + {"id": case.job}, + ) + == "pending" + ) + + +def test_report_execution_failed_attempt_replay_preserves_snapshot_and_history(database, case): + old = claim(database, case) + with transaction(database) as connection: + assert execution.fail(connection, old, "INTERNAL_ERROR") + before = records(database, case) + repository = JobRepository() + actor = admin(case) + with transaction(database) as connection: + task, report = repository.replay_source(connection, actor, case.job) + require_replay(actor, task, report, task["version"]) + result = repository.replay(connection, actor, task, "Retry frozen CSV", str(uuid4())) + assert result["state"] == "ready" and result["replay_generation"] == 1 + assert claim(database, case) is None + with transaction(database) as connection: + assert not execution.fail(connection, old, "INTERNAL_ERROR") + with pytest.raises(LeaseLost): + execution.commit_result(connection, old, artifact(old)) + new = claim(database, case, {**case.message, "replay_generation": 1, "dispatch_sequence": 2}) + assert new.attempt == 1 and new.token > old.token + assert new.input.snapshot == old.input.snapshot + with transaction(database) as connection: + execution.commit_result(connection, new, artifact(new)) + after = records(database, case) + assert all(entry in after["task_attempts"] for entry in before["task_attempts"]) + assert len(after["task_attempts"]) == 2 + assert row(database, "report_exports", case.export)["status"] == "ready" + + +def test_report_execution_recovered_winner_cannot_be_overwritten_by_old_upload(database, case): + old = claim(database, case) + orphan = artifact(old) + update(database, "UPDATE task_runs SET lease_until='1990-01-01' WHERE id=:id", id=case.job) + with transaction(database) as connection: + candidate = next(r for r in execution.expired_candidates(connection) if r["id"] == case.job) + assert execution.recover(connection, candidate) + update(database, "UPDATE task_runs SET available_at=UTC_TIMESTAMP(3) WHERE id=:id", id=case.job) + winner = claim(database, case) + with transaction(database) as connection: + execution.commit_result( + connection, winner, replace(artifact(winner), object_version="winner") + ) + before = records(database, case) + with transaction(database) as connection: + with pytest.raises(LeaseLost): + execution.commit_result(connection, old, orphan) + assert not execution.fail(connection, old, "INTERNAL_ERROR") + assert records(database, case) == before + assert row(database, "report_exports", case.export)["object_version"] == "winner" diff --git a/tests/persistence/test_report_export_mysql.py b/tests/persistence/test_report_export_mysql.py new file mode 100644 index 0000000..1867e0e --- /dev/null +++ b/tests/persistence/test_report_export_mysql.py @@ -0,0 +1,419 @@ +"""Report snapshot, authorization and atomic replay on disposable MySQL only.""" + +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta +from threading import Barrier +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import event, text + +from packages.domain.security import Principal, ServiceError +from packages.persistence import reports +from packages.persistence.dispatch import decoded, validate_dispatch +from packages.persistence.users import timestamp +from tests.persistence.factories import Graph, insert +from tests.persistence.test_identity_mysql import ORIGIN, assert_schema, headers, login +from tests.persistence.test_roles_mysql import command, create_user, lab_fixture, role_write + + +def body(labs): + now = datetime.utcnow() + return { + "format": "csv", + "filters": { + "laboratory_ids": labs, + "from": timestamp(now - timedelta(days=1)), + "to": timestamp(now + timedelta(days=1)), + "severity": [], + "finding_status": [], + }, + } + + +@pytest.fixture +def graph(transaction): + graph = Graph(transaction) + for item, run, fact, evaluation in ( + (graph.item, graph.run, graph.fact, graph.evaluation), + (graph.other_item, graph.other_run, graph.other_fact, graph.other_evaluation), + ): + transaction.execute( + text( + "UPDATE inspection_items SET current_run_id=:run," + "current_fact_revision_id=:fact,current_evaluation_id=:evaluation WHERE id=:id" + ), + {"id": item, "run": run, "fact": fact, "evaluation": evaluation}, + ) + return graph + + +def snapshot(connection, export_id): + return decoded( + connection.scalar( + text("SELECT snapshot FROM report_exports WHERE id=:id"), {"id": export_id} + ) + ) + + +def admin(graph): + return Principal(graph.user, graph.tenant, "synthetic", (("safety_admin", "tenant", None),)) + + +def create(connection, graph, request=None): + return reports.create(connection, admin(graph), request or body([graph.lab]), str(uuid4())) + + +def test_report_export_snapshot_and_outbox_binding(transaction, graph): + result = create(transaction, graph) + frozen = snapshot(transaction, result["id"]) + assert frozen["export_id"] == result["id"] and frozen["tenant_id"] == graph.tenant + assert frozen["laboratory_ids"] == [graph.lab] + assert len(frozen["rows"]) == 2 + finding = next(row for row in frozen["rows"] if row["finding_id"]) + assert finding["fact_revision_id"] == graph.fact and finding["run_id"] == graph.run + assert finding["rule_bundle_id"] == graph.rules + assert finding["task_status"] == "pending_dispatch" + assert {row["item_id"] for row in frozen["rows"]} == {graph.item, graph.other_item} + assert all(set(row) == set(frozen["columns"]) for row in frozen["rows"]) + envelopes = ( + transaction.execute( + text("SELECT event_type,payload FROM outbox_events WHERE tenant_id=:tenant"), + {"tenant": graph.tenant}, + ) + .mappings() + .all() + ) + assert {e["event_type"] for e in envelopes} == {"TaskDispatch", "ReportRequested"} + dispatch = next(decoded(e["payload"]) for e in envelopes if e["event_type"] == "TaskDispatch") + assert validate_dispatch(dispatch)["payload"] == {"export_id": result["id"]} + assert dispatch["task_id"] == result["job_id"] + transaction.execute( + text("UPDATE findings SET explanation='changed' WHERE id=:id"), {"id": graph.finding} + ) + assert snapshot(transaction, result["id"]) == frozen + + +def test_report_export_filters_exclude_history_and_keep_zero_finding_items(transaction, graph): + transaction.execute( + text("UPDATE findings SET severity='critical',status='confirmed' WHERE id=:id"), + {"id": graph.finding}, + ) + request = body([graph.lab]) + request["filters"]["severity"] = ["low"] + result = create(transaction, graph, request) + assert [r["item_id"] for r in snapshot(transaction, result["id"])["rows"]] == [graph.other_item] + transaction.execute( + text("UPDATE findings SET superseded_at=UTC_TIMESTAMP(3) WHERE id=:id"), + {"id": graph.finding}, + ) + result = create(transaction, graph) + assert all(r["finding_id"] is None for r in snapshot(transaction, result["id"])["rows"]) + + +def test_report_export_exact_10000_finding_limit(transaction, graph): + values = [ + { + "id": str(uuid4()), + "tenant": graph.tenant, + "item": graph.item, + "run": graph.run, + "fact": graph.fact, + "evaluation": graph.evaluation, + "fingerprint": uuid4().hex * 2, + } + for _ in range(10000) + ] + sql = text( + "INSERT INTO findings (id,tenant_id,item_id,run_id,fact_revision_id,rule_evaluation_id," + "rule_id,fingerprint,type,explanation,evidence) " + "VALUES (:id,:tenant,:item,:run,:fact,:evaluation,'synthetic'," + ":fingerprint,'synthetic','risk','[]')" + ) + transaction.execute(sql, values[:9999]) + result = create(transaction, graph) + assert len(snapshot(transaction, result["id"])["rows"]) == 10001 # Includes one empty item. + transaction.execute(sql, values[9999:]) + with pytest.raises(ServiceError) as error: + create(transaction, graph) + assert error.value.status == 422 + assert ( + transaction.scalar( + text("SELECT COUNT(*) FROM report_exports WHERE tenant_id=:tenant"), + {"tenant": graph.tenant}, + ) + == 1 + ) + + +@pytest.mark.parametrize( + "role,scope,expected", + [ + ("lab_manager", "own", 202), + ("viewer", "own", 403), + ("lab_manager", "other", 404), + ("safety_admin", "foreign_tenant", 404), + ], +) +def test_report_export_repository_scope(transaction, graph, role, scope, expected): + request = body([graph.lab]) + tenant = str(uuid4()) if scope == "foreign_tenant" else graph.tenant + roles = ( + ((role, "tenant", None),) + if role == "safety_admin" + else ((role, "laboratory", str(uuid4()) if scope == "other" else graph.lab),) + ) + actor = Principal(graph.user, tenant, "synthetic", roles) + if expected == 202: + result = reports.create(transaction, actor, request, str(uuid4())) + assert reports.get(transaction, actor, result["id"])["status"] == "queued" + else: + with pytest.raises(ServiceError) as error: + reports.create(transaction, actor, request, str(uuid4())) + assert error.value.status == expected + + +@pytest.fixture +def http_case(identity, database): + tenant, service, app = identity + lab = lab_fixture(database, tenant) + with TestClient(app, base_url=ORIGIN) as http: + session = login(http, tenant).json() + yield tenant, service, http, session, body([lab]) + + +def post(http, session, request, key="report-create-key"): + return http.post("/api/v1/reports/exports", json=request, headers=headers(session, key)) + + +def tables(database, tenant): + with database.connect() as connection: + return { + name: [ + dict(r) + for r in connection.execute( + text(f"SELECT * FROM {name} WHERE tenant_id=:tenant ORDER BY id"), + {"tenant": tenant}, + ).mappings() + ] + for name in ( + "report_exports", + "task_runs", + "task_attempts", + "outbox_events", + "audit_events", + "api_idempotency", + ) + } + + +def test_report_export_http_schema_idempotency_conflict_and_lab_manager(http_case, database): + tenant, _, http, session, request = http_case + user = create_user(http, session, "reporter") + assert ( + role_write( + http, + session, + user["id"], + command(role="lab_manager", lab=request["filters"]["laboratory_ids"][0]), + ).status_code + == 200 + ) + session = login(http, tenant, username="reporter").json() + first = post(http, session, request) + assert first.status_code == 202, first.text + assert_schema(first, "ReportExport") + before = tables(database, tenant["tenant_id"]) + assert post(http, session, request).content == first.content + assert tables(database, tenant["tenant_id"]) == before + assert post(http, session, {**request, "format": "pdf"}).status_code == 409 + report_id, job_id = first.json()["data"]["id"], first.json()["data"]["job_id"] + read = http.get(f"/api/v1/reports/exports/{report_id}") + assert read.status_code == 200 + assert_schema(read, "ReportExport") + assert_schema(http.get(f"/api/v1/jobs/{job_id}"), "Job") + with database.begin() as connection: + connection.execute( + text("DELETE FROM user_roles WHERE tenant_id=:tenant AND user_id=:id"), + {"tenant": tenant["tenant_id"], "id": user["id"]}, + ) + assert post(http, session, request).status_code == 404 + assert http.get(f"/api/v1/reports/exports/{report_id}").status_code == 404 + + +@pytest.mark.parametrize( + "statement", + [ + "INSERT INTO report_exports", + "INSERT INTO task_runs", + "INSERT INTO outbox_events", + "INSERT INTO audit_events", + "UPDATE api_idempotency", + ], +) +def test_report_export_create_write_failure_rolls_back(http_case, database, statement): + tenant, _, http, session, request = http_case + before = tables(database, tenant["tenant_id"]) + + def fail(connection, cursor, sql, params, context, many): + if sql.startswith(statement): + raise RuntimeError("injected report write failure") + + event.listen(database, "before_cursor_execute", fail) + try: + response = post(http, session, request) + finally: + event.remove(database, "before_cursor_execute", fail) + assert response.status_code == 500 + assert tables(database, tenant["tenant_id"]) == before + + +def make_failed(database, result): + with database.begin() as connection: + tenant = connection.scalar( + text("SELECT tenant_id FROM task_runs WHERE id=:id"), {"id": result["job_id"]} + ) + insert( + connection, + "task_attempts", + tenant_id=tenant, + task_id=result["job_id"], + fencing_token=4, + attempt=4, + status="failed", + error_code="INTERNAL_ERROR", + finished_at=datetime.utcnow(), + ) + connection.execute( + text( + "UPDATE report_exports SET status='failed',last_error_code='INTERNAL_ERROR'," + "version=3 WHERE id=:id" + ), + {"id": result["id"]}, + ) + connection.execute( + text( + "UPDATE task_runs SET state='dead_letter',attempt=4,version=7," + "fencing_token=4,last_error_code='INTERNAL_ERROR' WHERE id=:id" + ), + {"id": result["job_id"]}, + ) + + +def test_report_export_job_replay_and_list_are_atomic(http_case, database): + tenant, _, http, session, request = http_case + first = post(http, session, request) + assert first.status_code == 202, first.text + result = first.json()["data"] + make_failed(database, result) + before = tables(database, tenant["tenant_id"]) + listed = http.get( + "/api/v1/dead-letters", + params={ + "laboratory_id": request["filters"]["laboratory_ids"][0], + }, + ) + assert listed.status_code == 200, listed.text + assert listed.json()["data"]["items"][0]["id"] == result["job_id"] + path = f"/api/v1/dead-letters/{result['job_id']}/replay" + response = http.post( + path, + json={"expected_version": 7, "reason": "retry"}, + headers=headers(session, "report-replay-key"), + ) + assert response.status_code == 202, response.text + assert_schema(response, "Job") + assert response.json()["data"]["replay_generation"] == 1 + replayed = http.post( + path, + json={"expected_version": 7, "reason": "retry"}, + headers=headers(session, "report-replay-key"), + ) + assert replayed.content == response.content + after = tables(database, tenant["tenant_id"]) + assert after["report_exports"][0]["status"] == "queued" + assert after["report_exports"][0]["snapshot"] == before["report_exports"][0]["snapshot"] + assert after["task_runs"][0]["fencing_token"] == 5 + assert after["task_runs"][0]["attempt"] == 0 + assert after["task_attempts"] == before["task_attempts"] + assert len(after["outbox_events"]) == len(before["outbox_events"]) + 1 + + +@pytest.mark.parametrize("fault", ["version", "ready", "snapshot", "payload"]) +def test_report_export_replay_rejects_invalid_source(http_case, database, fault): + tenant, _, http, session, request = http_case + first = post(http, session, request) + assert first.status_code == 202, first.text + result = first.json()["data"] + make_failed(database, result) + if fault != "version": + sql = { + "ready": "UPDATE report_exports SET status='ready' WHERE id=:id", + "snapshot": "UPDATE report_exports SET snapshot='{}' WHERE id=:id", + "payload": "UPDATE task_runs SET payload='{}' WHERE id=:id", + }[fault] + with database.begin() as connection: + connection.execute( + text(sql), {"id": result["job_id"] if fault == "payload" else result["id"]} + ) + before = tables(database, tenant["tenant_id"]) + response = http.post( + f"/api/v1/dead-letters/{result['job_id']}/replay", + json={"expected_version": 6 if fault == "version" else 7, "reason": "retry"}, + headers=headers(session, "report-replay-key"), + ) + assert response.status_code == 409, response.text + assert tables(database, tenant["tenant_id"]) == before + + +def test_report_export_replay_failure_restores_failed_report(http_case, database): + tenant, _, http, session, request = http_case + first = post(http, session, request) + assert first.status_code == 202, first.text + result = first.json()["data"] + make_failed(database, result) + before = tables(database, tenant["tenant_id"]) + with patch("packages.persistence.jobs.append_audit", side_effect=RuntimeError("fault")): + response = http.post( + f"/api/v1/dead-letters/{result['job_id']}/replay", + json={"expected_version": 7, "reason": "retry"}, + headers=headers(session, "report-replay-key"), + ) + assert response.status_code == 500 + assert tables(database, tenant["tenant_id"]) == before + + +def test_report_export_concurrent_same_key_across_sessions(http_case, database): + tenant, service, http, _, request = http_case + barrier = Barrier(2) + with TestClient(http.app, base_url=ORIGIN) as second: + sessions = [login(client, tenant).json() for client in (http, second)] + tokens = [client.cookies.get("labsafe_session") for client in (http, second)] + + def create_one(index): + barrier.wait(timeout=10) + return http.app.state.reports.create( + tokens[index], + sessions[index]["data"]["csrf_token"], + "concurrent-export", + request, + str(uuid4()), + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + responses = list(pool.map(create_one, range(2))) + assert responses[0] == responses[1] and responses[0][0] == 202 + rows = tables(database, tenant["tenant_id"]) + assert len(rows["report_exports"]) == len(rows["task_runs"]) == 1 + assert len(rows["outbox_events"]) == 2 + + +def test_report_export_all_labs_required_even_when_snapshot_is_empty(transaction, graph): + another = graph.add("laboratories", college_id=graph.college) + result = create(transaction, graph, body([graph.lab, another])) + partial = Principal(graph.user, graph.tenant, "reader", (("viewer", "laboratory", graph.lab),)) + with pytest.raises(ServiceError) as error: + reports.get(transaction, partial, result["id"]) + assert error.value.status == 404 diff --git a/tests/persistence/test_unit.py b/tests/persistence/test_unit.py index be355eb..721f64b 100644 --- a/tests/persistence/test_unit.py +++ b/tests/persistence/test_unit.py @@ -20,10 +20,18 @@ ROOT = Path(__file__).resolve().parents[2] -def test_frozen_migration_matches_authoritative_design(): +def test_head_schema_matches_authoritative_design_and_preserves_initial_baseline(): assert model() == json.loads((ROOT / "contracts/data-model.json").read_text(encoding="utf-8")) + baseline = json.loads(MODEL_PATH.read_text(encoding="utf-8")) + head = model() + for name in ("object_version", "size_bytes"): + del head["tables"]["report_exports"]["columns"][name] + assert head == baseline snapshot = MODEL_PATH.with_suffix(".sql").read_text(encoding="utf-8").strip() - assert snapshot == (ROOT / "contracts/database-design.sql").read_text(encoding="utf-8").strip() + head_ddl = (ROOT / "contracts/database-design.sql").read_text(encoding="utf-8").strip() + additions = " `object_version` VARCHAR(200) NULL,\n `size_bytes` BIGINT UNSIGNED NULL,\n" + assert head_ddl.count(additions) == 1 + assert snapshot == head_ddl.replace(additions, "") @pytest.mark.parametrize("environment", ["production", "staging", "", "prod"]) diff --git a/tools/design/data_model.py b/tools/design/data_model.py index 9a13e94..e9a8539 100644 --- a/tools/design/data_model.py +++ b/tools/design/data_model.py @@ -65,7 +65,7 @@ def build_database(): table('task_attempts',{'task_id':uid(),'replay_generation':num(),'attempt':num('1'),'fencing_token':c('BIGINT UNSIGNED'),'lease_owner':uid(),'started_at':dt(),'finished_at':dt(True),'status':state('running','succeeded','failed','abandoned'),'error_code':s(80,True)},unique=[['tenant_id','task_id','replay_generation','attempt']]);link('task_attempts','task_id','task_runs') table('notifications',{'event_id':uid(),'recipient_id':uid(),'type':s(80),'resource_type':s(80),'resource_id':uid(),'read_at':dt(True)},unique=[['tenant_id','event_id','recipient_id']]);link('notifications','event_id','outbox_events');link('notifications','recipient_id','users') table('audit_events',{'actor_id':uid(True),'action':s(100),'resource_type':s(80),'resource_id':uid(),'changes':j(),'reason':s(2000),'request_id':uid()},indexes=[['tenant_id','resource_type','resource_id','created_at']],immutable=True);link('audit_events','actor_id','users') - table('report_exports',{'requested_by':uid(),'format':state('csv','pdf'),'filters':j(),'snapshot_at':dt(),'snapshot':j(),'status':state('queued','running','ready','failed','expired'),'object_key':s(1024,True),'checksum':c('CHAR(64)',True),'expires_at':dt(True),'last_error_code':s(80,True)});link('report_exports','requested_by','users') + table('report_exports',{'requested_by':uid(),'format':state('csv','pdf'),'filters':j(),'snapshot_at':dt(),'snapshot':j(),'status':state('queued','running','ready','failed','expired'),'object_key':s(1024,True),'checksum':c('CHAR(64)',True),'expires_at':dt(True),'last_error_code':s(80,True),'object_version':s(200,True),'size_bytes':c('BIGINT UNSIGNED',True)});link('report_exports','requested_by','users') table('dataset_versions',{'name':s(100),'checksum':c('CHAR(64)'),'manifest_key':s(1024),'authorization_ref':s(1000),'status':state('draft','frozen','retired')},unique=[['tenant_id','name']],immutable=True) table('dataset_images',{'dataset_id':uid(),'image_id':uid(),'scene_id':s(100),'split':state('train','validation','test'),'annotation_key':s(1024),'provenance':s(2000)},unique=[['tenant_id','dataset_id','image_id']],immutable=True);link('dataset_images','dataset_id','dataset_versions');link('dataset_images','image_id','asset_images') q=lambda x:chr(96)+x+chr(96)