From 86d0712cc7789345d724bcaa6ee4667a7c5ca197 Mon Sep 17 00:00:00 2001 From: SayQuan1 <15246906705@163.com> Date: Mon, 5 Oct 2026 17:12:15 +0800 Subject: [PATCH] feat: implement durable inspection workflow batches --- .env.example | 18 +- DEVELOPMENT.md | 96 +- apps/api/app/inference_runs.py | 50 + apps/api/app/jobs.py | 58 ++ apps/api/app/main.py | 19 +- apps/api/app/review.py | 173 ++++ apps/api/app/rules.py | 125 +++ apps/api/app/uploads.py | 29 +- apps/worker/app/main.py | 36 + apps/worker/dispatch.py | 122 +++ apps/worker/image_validation.py | 107 ++ apps/worker/inference_pipeline.py | 15 + apps/worker/rule_evaluation.py | 14 + apps/worker/run.py | 16 +- docs/02-architecture/05-durable-jobs.md | 10 + docs/03-interfaces/01-api-contract.md | 25 +- docs/08-delivery/01-implementation-plan.md | 54 +- docs/08-delivery/17-i03a1-durable-dispatch.md | 81 ++ docs/08-delivery/18-i03a2-image-execution.md | 97 ++ docs/08-delivery/19-i02f3-image-validation.md | 97 ++ docs/08-delivery/20-i03a3-image-job-replay.md | 97 ++ docs/08-delivery/21-i02f4-image-download.md | 94 ++ .../22-i02g1-submit-inference-enqueue.md | 17 + .../23-i03a2-inference-execution.md | 24 + docs/08-delivery/24-i03a2-rule-evaluation.md | 25 + docs/08-delivery/25-i02-rule-management.md | 31 + docs/08-delivery/26-i02-review.md | 25 + docs/08-delivery/README.md | 10 + docs/README.md | 6 +- packages/application/dispatch.py | 70 ++ packages/application/image_download.py | 50 + packages/application/image_validation.py | 77 ++ packages/application/inference_execution.py | 208 ++++ packages/application/inference_runs.py | 79 ++ packages/application/job_execution.py | 118 +++ packages/application/jobs.py | 103 ++ packages/application/rate_limit.py | 4 + packages/application/review.py | 138 +++ packages/application/rule_execution.py | 105 ++ packages/application/rules.py | 154 +++ packages/domain/image_download.py | 75 ++ packages/domain/image_validation.py | 75 ++ packages/domain/inference_execution.py | 151 +++ packages/domain/job_execution.py | 57 ++ packages/domain/job_replay.py | 32 + packages/domain/rule_execution.py | 38 + packages/persistence/dispatch.py | 269 +++++ packages/persistence/foundations.py | 15 +- packages/persistence/image_validation.py | 155 +++ packages/persistence/images.py | 12 + packages/persistence/inference_execution.py | 616 ++++++++++++ packages/persistence/inference_runs.py | 528 ++++++++++ packages/persistence/job_execution.py | 390 ++++++++ packages/persistence/jobs.py | 151 +++ packages/persistence/review.py | 917 ++++++++++++++++++ packages/persistence/rule_execution.py | 599 ++++++++++++ packages/persistence/rules.py | 422 ++++++++ packages/rules/evaluator.py | 401 ++++++++ packages/storage/s3.py | 199 +++- pyproject.toml | 1 + requirements/py311-business.txt | 1 + tests/business/test_dispatch.py | 178 ++++ tests/business/test_image_download.py | 265 +++++ tests/business/test_image_validation.py | 387 ++++++++ tests/business/test_inference_execution.py | 114 +++ tests/business/test_job_api.py | 256 +++++ tests/business/test_job_execution.py | 88 ++ tests/business/test_review_boundary.py | 136 +++ tests/business/test_rule_evaluator.py | 30 + tests/business/test_rules_boundary.py | 131 +++ tests/persistence/test_dispatch_mysql.py | 296 ++++++ .../persistence/test_image_download_mysql.py | 332 +++++++ .../test_image_validation_mysql.py | 374 +++++++ .../test_inference_execution_mysql.py | 233 +++++ .../test_inference_submit_mysql.py | 131 +++ tests/persistence/test_job_api_mysql.py | 545 +++++++++++ tests/persistence/test_job_execution_mysql.py | 493 ++++++++++ tests/persistence/test_review_mysql.py | 183 ++++ 78 files changed, 11896 insertions(+), 57 deletions(-) create mode 100644 apps/api/app/inference_runs.py create mode 100644 apps/api/app/jobs.py create mode 100644 apps/api/app/review.py create mode 100644 apps/api/app/rules.py create mode 100644 apps/worker/dispatch.py create mode 100644 apps/worker/image_validation.py create mode 100644 apps/worker/inference_pipeline.py create mode 100644 apps/worker/rule_evaluation.py create mode 100644 docs/08-delivery/17-i03a1-durable-dispatch.md create mode 100644 docs/08-delivery/18-i03a2-image-execution.md create mode 100644 docs/08-delivery/19-i02f3-image-validation.md create mode 100644 docs/08-delivery/20-i03a3-image-job-replay.md create mode 100644 docs/08-delivery/21-i02f4-image-download.md create mode 100644 docs/08-delivery/22-i02g1-submit-inference-enqueue.md create mode 100644 docs/08-delivery/23-i03a2-inference-execution.md create mode 100644 docs/08-delivery/24-i03a2-rule-evaluation.md create mode 100644 docs/08-delivery/25-i02-rule-management.md create mode 100644 docs/08-delivery/26-i02-review.md create mode 100644 packages/application/dispatch.py create mode 100644 packages/application/image_download.py create mode 100644 packages/application/image_validation.py create mode 100644 packages/application/inference_execution.py create mode 100644 packages/application/inference_runs.py create mode 100644 packages/application/job_execution.py create mode 100644 packages/application/jobs.py create mode 100644 packages/application/review.py create mode 100644 packages/application/rule_execution.py create mode 100644 packages/application/rules.py create mode 100644 packages/domain/image_download.py create mode 100644 packages/domain/image_validation.py create mode 100644 packages/domain/inference_execution.py create mode 100644 packages/domain/job_execution.py create mode 100644 packages/domain/job_replay.py create mode 100644 packages/domain/rule_execution.py create mode 100644 packages/persistence/dispatch.py create mode 100644 packages/persistence/image_validation.py create mode 100644 packages/persistence/inference_execution.py create mode 100644 packages/persistence/inference_runs.py create mode 100644 packages/persistence/job_execution.py create mode 100644 packages/persistence/jobs.py create mode 100644 packages/persistence/review.py create mode 100644 packages/persistence/rule_execution.py create mode 100644 packages/persistence/rules.py create mode 100644 packages/rules/evaluator.py create mode 100644 tests/business/test_dispatch.py create mode 100644 tests/business/test_image_download.py create mode 100644 tests/business/test_image_validation.py create mode 100644 tests/business/test_inference_execution.py create mode 100644 tests/business/test_job_api.py create mode 100644 tests/business/test_job_execution.py create mode 100644 tests/business/test_review_boundary.py create mode 100644 tests/business/test_rule_evaluator.py create mode 100644 tests/business/test_rules_boundary.py create mode 100644 tests/persistence/test_dispatch_mysql.py create mode 100644 tests/persistence/test_image_download_mysql.py create mode 100644 tests/persistence/test_image_validation_mysql.py create mode 100644 tests/persistence/test_inference_execution_mysql.py create mode 100644 tests/persistence/test_inference_submit_mysql.py create mode 100644 tests/persistence/test_job_api_mysql.py create mode 100644 tests/persistence/test_job_execution_mysql.py create mode 100644 tests/persistence/test_review_mysql.py diff --git a/.env.example b/.env.example index b5ec26e..7503553 100644 --- a/.env.example +++ b/.env.example @@ -3,16 +3,24 @@ API_HOST=127.0.0.1 API_PORT=8000 AI_INFERENCE_HOST=127.0.0.1 AI_INFERENCE_PORT=8001 +AI_INFERENCE_URL=http://127.0.0.1:8001 REDIS_URL=redis://localhost:6379/0 -# Identity and foundation APIs are opt-in; use local HTTPS with this exact origin. +# Development validate_image dispatch and opt-in general consumer (separate processes). +WORKER_DISPATCH_ENABLED=0 +WORKER_IMAGE_VALIDATION_ENABLED=0 +WORKER_INFERENCE_ENABLED=0 +WORKER_RULE_EVALUATION_ENABLED=0 +# Identity, foundation and image-job query APIs are opt-in; use this exact HTTPS origin. API_IDENTITY_ENABLED=0 PUBLIC_ORIGIN=https://labsafe.example.org CSRF_KEY_FILE=.local-secrets/csrf-key DATABASE_URL_FILE=.local-secrets/database-url DATABASE_SCHEMA=labsafe_dev_local -# Dev/test upload grants, validation acceptance and image metadata only. Requires a private, -# versioned bucket, least-privilege API credentials and same-origin HTTPS:443. No consumer yet. +# Dev/test upload grants, validation acceptance, image metadata and download signing. Requires a private, +# versioned bucket, least-privilege API credentials and same-origin HTTPS:443. +# New manual image-job replay also requires this API storage configuration. +# Image downloads use the same API credentials with O/A GetObjectVersion permission. API_UPLOADS_ENABLED=0 S3_ENDPOINT=http://127.0.0.1:9000 S3_PUBLIC_ENDPOINT=https://labsafe.example.org @@ -20,6 +28,10 @@ S3_BUCKET=labsafe-private S3_REGION=us-east-1 S3_ACCESS_KEY_FILE=.local-secrets/s3-api-access-key S3_SECRET_KEY_FILE=.local-secrets/s3-api-secret-key +# General Worker credentials: staging GET/version; original/analysis GET/version + PUT. +# No Delete, ListBucket or administration; never fall back to API credentials. +S3_WORKER_ACCESS_KEY_FILE=.local-secrets/s3-worker-access-key +S3_WORKER_SECRET_KEY_FILE=.local-secrets/s3-worker-secret-key AI_MODE=mock AI_TOKEN_FILE=.local-secrets/ai-token diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d7826ee..aeee9fe 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,5 +1,5 @@ -# LabSafe 开发指南（I-01A–I-02F2） +# LabSafe 开发指南（I-01A–I-02F4 / I-03A1–A3） ## 1. 本阶段边界 @@ -15,8 +15,11 @@ I-02C 接通角色列表、授予、撤销 3 个 API，覆盖 scope、用户版 I-02D 接通学院、实验室、位置、模板创建/发布/克隆及巡检草稿 16 个 API；可通过真实数据库完成组织→模板→草稿链路。 I-02E 接通巡检项列表、详情 2 个 API；提供同源守卫探测基础，未实现的写动作不显示为可执行能力。 I-02F1 新增可选 createUpload；实现上传授权、权限/额度守卫、真实 SDK 本地签名和固定版本读取/SHA 适配基础。 -I-02F2 新增 completeUpload/getImage，累计 31 个公共业务 API；上传完成只受理为 validating，并原子登记验证任务/TaskDispatch。图片下载、图像验证消费及 ready 提交仍未实现。 -尚不包含完整巡检业务仓储/API、持久任务闭环、真实图像处理、训练权重或生产部署。 +I-02F2 新增 completeUpload/getImage；上传完成受理为 validating，并原子登记验证任务/TaskDispatch。 +I-03A1/A2 接通上传验证任务持久调度、租约、围栏及恢复；I-02F3 接通可选 general Worker、真实 Pillow 图像处理与 ready/rejected 事务。 +I-03A3 新增 getJob/listDeadLetters/replayJob 的 validate_image 分支，累计注册 34 个公共业务 API；管理员重放保留固定输入和历史。 +I-02F4 新增 downloadImage，累计注册 35 个公共业务 API；ready 分析图按实验室 READ 下载，原图额外 Admin 与审计；签名固定版本、60 秒。 +尚不包含 crop/报告下载、完整巡检业务 API、其他任务类型/事件 inbox/AI 闭环、训练权重或生产部署；真实 MinIO/IAM/HTTPS 验收仍待完成。 未开启身份 API 时，/ready 只检查 I-01A 进程配置；开启后还检查迁移版本和 Redis 连通性，但仍不是完整系统就绪。开启上传也不会使 /ready 检查真实桶、IAM、签名请求或图像验证。 所有后端进程和前端构建均拒绝 production；AI_MODE 只接受 mock。 @@ -262,10 +265,87 @@ PUT 成功不等于图像有效或业务完成。后续完成/查询见 3.10， 首次完成要求 grant 未过期、owner 仍可采集、请求 hash 与 grant 声明一致、HEAD 大小/MIME 匹配。HEAD 元数据中的 SHA 不可信；validating 时的 original_sha256 仍为声明值，真实字节 SHA/解码由后续 Worker 验证。文件被重新 PUT 不会改写已固定的版本。 -本批没有 publisher/sweeper/验证 consumer，任务停留在数据库 ready、图像停留 validating，不应持续轮询期待本版本自行完成。staging 的 24 小时生命周期仍可能使未消费输入过期；启用真实验证消费前仅使用可丢弃合成测试对象，不能积压实际业务证据。关闭 API 开关不删除已登记任务；没有实现后台取消或清理功能。 +F2 自身只受理；I-03A1/A2 调度与恢复入口见 3.11，I-02F3 可选验证消费者见 3.12。消费者默认关闭，未启动时图像停留 validating。staging 的 24 小时生命周期可能使未消费输入过期；开发联调使用可丢弃合成图片，不能积压实际业务证据。关闭 API 开关不删除已登记任务；没有实现后台取消或清理功能。 验收、精确重放语义及后续关闭清单见 [I-02F2 验收](docs/08-delivery/16-i02f2-upload-completion-api.md)。真实 IAM/MinIO/nginx/TLS 联调仍待完成，TestClient/Stubber 不替代这些证据。 +### 3.11 可选上传验证任务调度（I-03A1） + +本批接通 validate_image TaskDispatch 的 MySQL → Redis 发布和补发，不接图像验证消费。沿用业务依赖与数据库 secret、REDIS_URL，新增 `WORKER_DISPATCH_ENABLED=1` 显式启用；默认关闭且拒绝 production。仅使用专用 dev/test 数据库与可丢弃 Redis。 + +~~~powershell +$env:APP_ENV = 'dev' +$env:WORKER_DISPATCH_ENABLED = '1' +.\.venv-business\Scripts\python.exe -m apps.worker.dispatch publisher --once +.\.venv-business\Scripts\python.exe -m apps.worker.dispatch sweeper --once +~~~ + +去掉 `--once` 可在两个终端分别运行循环。publisher 每轮最多处理 100 条（逐条领取），每轮等待 1 秒；sweeper 每轮回收/补发各最多 100 条，等待 15 秒。发布使用事务外独立子进程、20 秒预算和 10 秒续租。调度连接为 READ COMMITTED，不影响 API 查询快照。 + +消息写入 `q.general`；默认 fixture Worker 不注册 `labsafe.tasks.dispatch`，只有按 3.12 显式启用消费者才处理图像。不要让默认 fixture Worker 手动订阅该队列。停止调度进程、恢复开关 0 可回退，保留数据库调度历史。精确范围、故障测试和后续关闭清单见 [I-03A1 验收](docs/08-delivery/17-i03a1-durable-dispatch.md)。 + +I-03A2 继续沿用此入口：sweeper 先回收最多 100 个过期 validate_image 执行租约，再执行 A1 发布回收/补发；日志增加 execution_recovered。私有 `ImageExecution.execute(message, prepare, write_result)` 包含独立心跳与数据库围栏，现由 F3 服务端处理器调用；不要直接写脚本绕过结果事务。技术失败和内容拒绝的差异见 [执行基础验收](docs/08-delivery/18-i03a2-image-execution.md)。 + +### 3.12 图像验证 general Worker（I-02F3） + +重新安装 `requirements/dev.txt`（业务环境增加 Pillow 11.3.0）。沿用数据库/Redis/S3 endpoint、PUBLIC_ORIGIN 配置，另用 `S3_WORKER_ACCESS_KEY_FILE` 与 `S3_WORKER_SECRET_KEY_FILE` 指向独立 general Worker secret 文件；缺失即失败，不读取 API secret 替代。凭据仅允许 S 的 GET/GetVersion、O/A 的 GET/GetVersion/PUT，CopyObject 使用准确源版本；不授予 Delete/ListBucket/管理权限。 + +~~~powershell +$env:APP_ENV = 'dev' +$env:WORKER_IMAGE_VALIDATION_ENABLED = '1' +$env:S3_WORKER_ACCESS_KEY_FILE = '.local-secrets/s3-worker-access-key' +$env:S3_WORKER_SECRET_KEY_FILE = '.local-secrets/s3-worker-secret-key' +.\.venv-business\Scripts\python.exe -m apps.worker.run --check +.\.venv-business\Scripts\python.exe -m apps.worker.run +~~~ + +入口自动订阅 `q.general`，使用 solo/concurrency=1，以允许每任务 spawn 独立处理子进程；不要改成 Celery 的 daemon prefork pool。默认开关 0 仍只订阅 fixture 的 `celery` 队列。production 继续禁止，`--check` 检查配置不证明网络连通或 IAM 正确。另两个终端按 3.11 启动 publisher 与 sweeper；三者是不同进程，API 完成接口保持异步 202。 + +GET 固定 staging 版本并实算 SHA 后，用真实 Pillow 解码 JPEG/PNG/WebP，拒绝动画/格式伪装/截断和超限尺寸；仅执行一次 EXIF 旋转，输出去元数据 RGB PNG，不缩放分析图。事务外写 O/A；既有对象必须实算相同 SHA 才复用其准确版本，冲突不会覆盖。子进程总预算 120 秒，租约每 10 秒独立续 60 秒，失去租约或超时 kill/join；提交前重验围栏与输入。允许重复计算，只有当前租约能登记结果。 + +内容失败原子登记 image/upload=rejected、归属状态和审计；任务 succeeded 表示验证处理完成，不表示图片合格。仅成功 ready 产生 ImageValidated Outbox；技术失败保留 validating 并按持久任务重试/终止。事件发布/inbox 下游还未实现，ImageValidated 暂存 DB，不自动启动推理。 + +停止领取并允许当前任务在预算内排空；强制终止后由 sweeper 回收。关闭消费者/publisher/sweeper 并恢复开关 0 可回退；不删任务、图片或审计。未登记 O/A 是孤儿，后续清理须逐个精确版本核查引用，不在本批授予删除权限。最大 40M 像素会产生较大的解码内存和 PNG，输入 15 MiB 限制不是内存上限；当前一次一个任务，生产资源限额另行验收。范围、测试和真实存储验收清单见 [I-02F3 验收](docs/08-delivery/19-i02f3-image-validation.md)。 + +### 3.12.1 推理 pipeline Worker（I-03A2） + +推理消费者仍为 dev/test opt-in。先启动 AI fixture，再设置 `WORKER_INFERENCE_ENABLED=1`；它复用 q.general 的持久任务消息，调用 `/internal/inference/v1/quality` 和 `/runs`，结果由 MySQL 围栏事务接收。`facts_ready` 会原子创建 `fact_revisions` 和 `rule_evaluation` task；设置 `WORKER_RULE_EVALUATION_ENABLED=1` 后由同一 general Worker 执行规则快照评估。必须同时运行 publisher 和 sweeper。当前批次收敛质量重拍、事实修订、规则 findings 和待人工复核状态，rules 管理/report 任务仍未接通。详细范围见 [推理执行验收](docs/08-delivery/23-i03a2-inference-execution.md) 与 [规则评估验收](docs/08-delivery/24-i03a2-rule-evaluation.md)。 + +~~~powershell +$env:APP_ENV = 'dev' +$env:WORKER_INFERENCE_ENABLED = '1' +$env:AI_INFERENCE_URL = 'http://127.0.0.1:8001' +\.\.venv-business\Scripts\python.exe -m apps.worker.run --check +~~~ + +### 3.13 图像任务查询与管理员重放（I-03A3） + +沿用 `API_IDENTITY_ENABLED=1` 的数据库、Redis、同源 session 配置。GET `/api/v1/jobs/{id}` 需要图像所属实验室的 READ 权限；GET `/api/v1/dead-letters` 仅 safety_admin，支持 page/page_size/laboratory_id，列出 validate_image 的 failed 和 dead_letter 技术终态。Job 使用白名单字段，不返回 payload、对象 key、租约或围栏。其他任务类型当前返回 404，不代表整个任务平台已接通。 + +POST `/api/v1/dead-letters/{id}/replay` 仅 safety_admin，沿用 Origin、Content-Type=application/json、X-CSRF-Token 和 Idempotency-Key 请求头。先 GET Job，使用其 version（不是 Image.version）作为 expected_version；请求体如下： + +~~~json +{"expected_version": 3, "reason": "对象存储访问已恢复，人工重新执行验证"} +~~~ + +成功返回 202 的 Job，其中 state=ready、attempt=0、replay_generation 增加。任务、dispatch、审计、幂等响应原子登记，旧 attempts 保留；由现有 publisher/general Worker/sweeper 异步执行，不直接把图像改成 ready。只允许重放仍处于 validating、归属仍允许采集的固定输入；内容拒绝任务为 succeeded，应重新上传图片。 + +新重放还需要 `API_UPLOADS_ENABLED=1` 及 3.9 的 API S3 secret/endpoint 配置，使用原 staging VersionId 做事务外 HEAD；缺少存储配置返回 503。缺失版本返回 409 STATE_CONFLICT，应重新上传，不会续签或回退到 latest。HEAD 不证明 SHA/图片合格，F3 仍执行准确 GET、实算 SHA 和解码。相同 key/body 返回原始 202，但每次重新检查当前身份/Admin；不重复 HEAD、不增加 generation。相同 key、不同请求体返回 409。 + +无新增配置或迁移。停止 API 可暂停新人工重放，保留任务/审计/历史；已受理的任务仍由 Worker 执行，停止异步处理按 3.11/3.12 排空。精确事务与验收见 [I-03A3 验收](docs/08-delivery/20-i03a3-image-job-replay.md)。 + +### 3.14 受控图像下载（I-02F4） + +沿用 `API_IDENTITY_ENABLED=1`、`API_UPLOADS_ENABLED=1` 与 3.9 的 S3 配置，无新增 secret。API 凭据需具有获准租户/实验室 O/A 前缀的 GetObjectVersion 权限；不要为浏览器或 API 增加 List/Delete 权限。Worker 仍使用独立凭据。 + +GET `/api/v1/images/{id}/download` 默认 analysis，`?variant=original` 需要 safety_admin；两者先核图像实验室 READ。唯一 query 为单个 variant，不接受 object_key/versionId/tenant_id。GET 使用现有 session，无 Idempotency-Key/CSRF 命令头；URL 只通过同源受认证 API 返回，不重定向。 + +响应为 `{data:{url,expires_at},request_id}`，URL 使用同源 HTTPS、path-style SigV4、准确 VersionId，签发时有效期 60 秒。原样使用完整 URL，不替换 Host/路径/参数；图片应使用页面 no-referrer 策略，不复制 URL 至日志、统计或持久缓存。API 响应设置 Cache-Control=no-store、Referrer-Policy=no-referrer。 + +仅 ready 且所选 variant 的规范 O/A key、SHA、准确版本齐全时签发；validating/rejected/deleted 返回 409。普通读者只获分析图；原图每次签发写 image.download_original 审计，失败则整个事务回滚、返回错误，不能收到 grant。签发在身份/角色/session/图片锁内进行，使用固定凭据做本地签名，不在事务内 HEAD/GET。下载不会修改图像、上传、任务或事件。 + +下载签发共用 120 次/分钟用户配额，不占 60 次/分钟写配额；Redis 故障返回 503，无普通元数据 GET 的本地降级。撤权后禁止再签发，已经签发的 URL 最长仍可用至 60 秒到期；不能宣称随 session 立即撤销。签发不检查存储网络和对象存在性，准确版本已丢失时对象 GET 可能失败，不能回退 latest 或修改 DB ready 状态。关闭上传开关并重启 API 可停止签发；保留历史审计与对象。证据与真实部署待验收项见 [I-02F4 验收](docs/08-delivery/21-i02f4-image-download.md)。 + ## 4. 验证 根目录、业务环境执行：领域守卫也使用业务环境，不向 AI 环境引入业务授权或持久化依赖。 @@ -291,7 +371,7 @@ AI 环境执行： I-01A 的业务/AI/协议测试自行在临时目录生成令牌和合成请求，不要求预设 APP_ENV，也不连接 Redis/MySQL/对象存储。 包含实际 API/AI 子进程 HTTP smoke，不只是 import 非空检查。 -I-01B–I-02F2 的真实数据库测试必须运行 3.5 的隔离启动器；直接 pytest tests/persistence 会跳过未配置的 MySQL 用例，不能把跳过算作数据库通过。上传测试的本地 SDK 签名/Stubber 不连接真实 S3，也不能替代 UP-03/SEC-03 部署验收。 +I-01B–I-02F4/I-03A 的真实数据库测试必须运行 3.5 的隔离启动器；直接 pytest tests/persistence 会跳过未配置的 MySQL 用例，不能把跳过算作数据库通过。图像处理测试使用真实 Pillow；SDK Stubber 和本地 SigV4 不连接真实 S3，不能替代 UP-03/SEC-03 部署验收。 设计校验使用独立工具环境，安装 tools/design/requirements.txt 后执行： @@ -312,12 +392,12 @@ CI 必须先 --check，不能先生成来掩盖漂移。设计校验不是应用 I-01B 的实际结果与后续边界见 [验收记录](docs/08-delivery/08-i01b-acceptance.md)。I-01C 会话、RBAC、幂等与租户事务实现及验收范围见 [I-01C 验收记录](docs/08-delivery/09-i01c-acceptance.md)。 -截至 2026-10-01，I-01A PR #5、I-01B PR #7、I-01C PR #8 均已合并。本轮基于 #8 合并后的 main 进入 I-02；数据库/安全变更仍需独立评审，不直接推送 main。 -I-02A 的领域守卫边界见 [领域命令守卫验收](docs/08-delivery/10-i02a-domain-guards.md)；本批最新身份 API、事务和故障验证见 [I-02B 验收](docs/08-delivery/11-i02b-identity-api.md)。本批未推送，不能将旧 PR 的 CI 成绩作为新分支已通过。 +截至已核验的历史状态，I-01A PR #5、I-01B PR #7、I-01C PR #8、I-02A–F2 PR #9 均已合并。当前在 `wsq/i-03a-durable-dispatch` 保留 A1/A2/F3/A3/F4 本地改动；本轮未提交、Push、创建 PR 或重新核验远程状态。数据库/安全变更仍需独立评审，不直接推送 main。 +I-02A 的领域守卫边界见 [领域命令守卫验收](docs/08-delivery/10-i02a-domain-guards.md)；身份 API、事务和历史故障验证见 [I-02B 验收](docs/08-delivery/11-i02b-identity-api.md)。当前 A1/A2/F3/A3/F4 未推送，不能将旧 PR 的 CI 成绩作为新分支已通过。 I-02C 的角色接口、作用域、版本与会话语义见 [角色 API 验收](docs/08-delivery/12-i02c-role-api.md)。角色请求的 expected_version 来自目标 User.version，laboratory_id 必须显式传 UUID/null；成功改权会撤销目标全部会话，包括操作者修改自身角色时的当前会话。三接口与 I-02B 共用开关和安全请求头，无新增配置。 I-02D 的接口、事务与本轮会话锁修复见 [基础 API 验收](docs/08-delivery/13-i02d-foundation-api.md)。实施计划的 I-0 表格同步记录本地验证状态，不把未提交工作记作远程 CI 或独立批准。 I-02E 的两个巡检项查询、同源探测基础与能力门禁见 [巡检项查询验收](docs/08-delivery/14-i02e-item-query-api.md)；这不代表完整写动作投影已接通。 I-02F1 的上传授权、S3 配置/签名、固定版本适配与实际测试见 [上传授权验收](docs/08-delivery/15-i02f1-upload-grant-api.md)，默认关闭，仅 dev/test；CI 的 AI 依赖隔离清单同时禁止 boto3/botocore，本地通过不代表远程 CI 已运行。 I-02F2 的完成受理/图像查询和原子任务写入见 [完成受理验收](docs/08-delivery/16-i02f2-upload-completion-api.md)；不把初始 task/dispatch 记录等同于持久任务闭环。 -下一批优先 I-03A：落地 publisher、任务领取/lease/fencing、重试和 sweeper 共用基础，再由 I-02F3 接入 general 图像验证消费及真实存储联调；此前不标 ready。后续接提交/重试/完成及事实、评估、复核与整改完整事务。真实模型主线按 I-ML-01 推进，不以基础 API、查询或纯领域测试代替模型和生产验收。 +I-03A1/A2 已接调度、上传验证执行租约与技术失败恢复；I-02F3 已补 general 消费者、真实图像解码和完整 ready/rejected 事务，见 [图像验证验收](docs/08-delivery/19-i02f3-image-validation.md)。I-03A3 新增 3 个图像任务查询/人工重放 API，见 [重放验收](docs/08-delivery/20-i03a3-image-job-replay.md)；I-02F4 新增受控图像下载，见 [本批验收](docs/08-delivery/21-i02f4-image-download.md)。真实 MinIO/IAM/nginx/TLS 部署验收仍未完成。其他任务类型及其 replay、领域事件发布/inbox 仍待实现。后续接提交/重试/完成及事实、评估、复核与整改完整事务。真实模型主线按 I-ML-01 推进，不以局部测试代替模型和生产验收。 diff --git a/apps/api/app/inference_runs.py b/apps/api/app/inference_runs.py new file mode 100644 index 0000000..0d6f0de --- /dev/null +++ b/apps/api/app/inference_runs.py @@ -0,0 +1,50 @@ +"""Inspection item submission route.""" + +from typing import Annotated, Literal +from uuid import UUID + +from fastapi import APIRouter, Request +from pydantic import Field + +from apps.api.app.foundations import Identifier +from apps.api.app.identity import Body, CanonicalJSONResponse +from packages.application.inference_runs import InferenceRunApplication +from packages.domain.security import ServiceError +from packages.persistence.security import SESSION_COOKIE + + +class ImageSelectionBody(Body): + image_id: Identifier + role: Literal["overview", "detail"] + parent_image_id: Identifier | None + + +class ItemSubmitBody(Body): + expected_version: Annotated[int, Field(ge=1, le=2147483647)] + images: list[ImageSelectionBody] = Field(min_length=1, max_length=3) + + +def mount_inference_runs(app, identity): + app.state.inference_runs = InferenceRunApplication(identity) if identity is not None else None + router = APIRouter(prefix="/api/v1") + + @router.post( + "/inspection-items/{item_id}/submit", status_code=202, operation_id="submitInspectionItem" + ) + def submit_item(item_id: UUID, body: ItemSubmitBody, request: Request): + if request.query_params: + raise ServiceError("VALIDATION_ERROR", 422, "Submit takes no query parameters") + if app.state.inference_runs is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Inference service not configured") + value = body.model_dump(mode="json") + status, response = app.state.inference_runs.submit( + request.cookies.get(SESSION_COOKIE), + request.headers["x-csrf-token"], + request.headers["idempotency-key"], + str(item_id), + value, + request.state.request_id, + ) + return CanonicalJSONResponse(response, status_code=status) + + app.include_router(router) diff --git a/apps/api/app/jobs.py b/apps/api/app/jobs.py new file mode 100644 index 0000000..dc33c09 --- /dev/null +++ b/apps/api/app/jobs.py @@ -0,0 +1,58 @@ +"""Strict job metadata and administrator replay adapters; validate_image branch only.""" + +from uuid import UUID + +from fastapi import APIRouter, Request + +from apps.api.app.identity import CanonicalJSONResponse, VersionCommand, pagination +from packages.application.jobs import JobApplication +from packages.domain.security import ServiceError +from packages.persistence.security import SESSION_COOKIE + + +def mount_jobs(app, identity, storage): + app.state.jobs = JobApplication(identity, storage) if identity is not None else None + router = APIRouter(prefix="/api/v1") + + def context(request, allowed=()): + if any( + k not in allowed or len(request.query_params.getlist(k)) != 1 + for k in request.query_params + ): + raise ServiceError("VALIDATION_ERROR", 422, "Unexpected or duplicate query parameter") + if app.state.jobs is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Job service not configured") + return app.state.jobs + + @router.get("/jobs/{job_id}", operation_id="getJob") + def get_job(job_id: UUID, request: Request): + return context(request).read( + "get", request.cookies.get(SESSION_COOKIE), request.state.request_id, job_id=str(job_id) + ) + + @router.get("/dead-letters", operation_id="listDeadLetters") + def dead_letters(request: Request, laboratory_id: UUID | None = None): + service = context(request, ("page", "page_size", "laboratory_id")) + page, size = pagination(request) + return service.read( + "dead_letters", + request.cookies.get(SESSION_COOKIE), + request.state.request_id, + page=page, + page_size=size, + laboratory_id=str(laboratory_id) if laboratory_id else None, + ) + + @router.post("/dead-letters/{job_id}/replay", status_code=202, operation_id="replayJob") + def replay(job_id: UUID, body: VersionCommand, request: Request): + status, response = context(request).replay( + request.cookies.get(SESSION_COOKIE), + request.headers["x-csrf-token"], + request.headers["idempotency-key"], + str(job_id), + body.model_dump(), + request.state.request_id, + ) + return CanonicalJSONResponse(response, status_code=status) + + app.include_router(router) diff --git a/apps/api/app/main.py b/apps/api/app/main.py index 45aa56b..683330a 100644 --- a/apps/api/app/main.py +++ b/apps/api/app/main.py @@ -1,4 +1,4 @@ -"""Development-only business API with opt-in upload validation acceptance.""" +"""Development-only business API with image-job queries and opt-in storage replay.""" import os from contextlib import asynccontextmanager @@ -7,7 +7,11 @@ from apps.api.app.foundations import mount_foundations from apps.api.app.identity import configured_identity, mount_identity +from apps.api.app.inference_runs import mount_inference_runs from apps.api.app.item_queries import mount_item_queries +from apps.api.app.jobs import mount_jobs +from apps.api.app.review import mount_review +from apps.api.app.rules import mount_rules from apps.api.app.uploads import mount_uploads from packages.shared.environment import development_environment @@ -54,7 +58,11 @@ async def lifespan(app): mount_identity(app, identity, public_origin) mount_foundations(app, identity) mount_item_queries(app, identity) + mount_inference_runs(app, identity) mount_uploads(app, identity, storage) + mount_jobs(app, identity, storage) + mount_rules(app, identity) + mount_review(app, identity) @app.get("/health") def health() -> dict[str, str | bool]: @@ -69,9 +77,12 @@ def ready() -> dict[str, str | bool]: "environment": environment, "is_simulated": True, "scope": ( - "I-02B/C identity, I-02D foundations, I-02E item queries " - "and optional I-02F1/F2 upload signing and validation acceptance only; " - "storage, workflow and real-model readiness not implemented" + "I-02B/C identity, I-02D foundations, I-02E item queries, " + "I-03A3 image-job queries and optional replay, I-02G1 submit enqueue, " + "optional I-02F1/F2 upload signing and validation acceptance only, " + "optional I-02F4 image download signing, and development rule " + "configuration lifecycle; " + "storage, inference execution and real-model readiness not implemented" ), } return { diff --git a/apps/api/app/review.py b/apps/api/app/review.py new file mode 100644 index 0000000..73aa6d3 --- /dev/null +++ b/apps/api/app/review.py @@ -0,0 +1,173 @@ +"""Facts and finding review API adapters.""" + +from typing import Annotated, Literal +from uuid import UUID + +from fastapi import APIRouter, Request +from pydantic import Field + +from apps.api.app.foundations import Identifier +from apps.api.app.identity import Body, CanonicalJSONResponse, VersionCommand, pagination +from packages.application.review import ReviewApplication +from packages.domain.security import ServiceError +from packages.persistence.security import SESSION_COOKIE + + +class EvidenceRefBody(Body): + image_id: Identifier + detection_id: Identifier | None + crop_id: Identifier | None + + +class EntityFactBody(Body): + detection_id: Identifier + entity_id: Identifier | None + resolution: Literal["resolved", "candidate", "unknown"] + source: Literal["model", "human"] + evidence: Annotated[list[EvidenceRefBody], Field(min_length=1, max_length=10)] + + +class RelationFactBody(Body): + source_detection_id: Identifier + target_detection_id: Identifier + relation: Literal["adjacent", "not_adjacent", "unknown"] + same_location: bool | None + source: Literal["model", "human"] + evidence: Annotated[list[EvidenceRefBody], Field(min_length=1, max_length=10)] + + +class DateFactBody(Body): + detection_id: Identifier + kind: Literal["expiry", "production", "opened", "unknown"] + value: Annotated[str, Field(pattern=r"^\d{4}-\d{2}-\d{2}$")] | None + source: Literal["model", "human"] + evidence: Annotated[list[EvidenceRefBody], Field(min_length=1, max_length=10)] + + +class FactsEditBody(Body): + expected_version: Annotated[int, Field(ge=1, le=2147483647)] + entities: Annotated[list[EntityFactBody], Field(max_length=100)] + relations: Annotated[list[RelationFactBody], Field(max_length=200)] + dates: Annotated[list[DateFactBody], Field(max_length=100)] + reason: Annotated[str, Field(min_length=1, max_length=2000, pattern=r"\S")] + + +class FindingUncertainBody(VersionCommand): + reason_code: Literal[ + "blur", + "occluded", + "label_unreadable", + "rule_missing", + "insufficient_evidence", + "other", + ] + + +def mount_review(app, identity): + app.state.review = ReviewApplication(identity) if identity is not None else None + router = APIRouter(prefix="/api/v1") + + def context(request, allowed=()): + if any( + key not in allowed or len(request.query_params.getlist(key)) != 1 + for key in request.query_params + ): + raise ServiceError("VALIDATION_ERROR", 422, "Unexpected or duplicate query parameter") + if app.state.review is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Review service not configured") + return app.state.review + + def read(request, kind, resource_id=None): + allowed = ( + () + if resource_id + else ( + "page", + "page_size", + "laboratory_id", + "current_only", + "status", + "severity", + ) + ) + service = context(request, allowed) + page, size = pagination(request) + laboratory_id = request.query_params.get("laboratory_id") + if laboratory_id is not None: + try: + laboratory_id = str(UUID(laboratory_id)) + except (TypeError, ValueError): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid laboratory_id") from None + raw_current = request.query_params.get("current_only", "true") + if raw_current not in {"true", "false"}: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid current_only") + status = request.query_params.get("status") + if status is not None and status not in { + "needs_review", + "confirmed", + "rejected", + "cannot_determine", + "dispatched", + "closed", + }: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid finding status") + severity = request.query_params.get("severity") + if severity is not None and severity not in {"low", "medium", "high", "critical"}: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid finding severity") + return service.read( + kind, + request.cookies.get(SESSION_COOKIE), + request.state.request_id, + resource_id=str(resource_id) if resource_id else None, + laboratory_id=laboratory_id, + page=page, + page_size=size, + current_only=raw_current == "true", + status=status, + severity=severity, + ) + + def write(request, operation, resource_id, body): + service = context(request) + status, response = service.write( + operation, + request.cookies.get(SESSION_COOKIE), + request.headers["x-csrf-token"], + request.headers["idempotency-key"], + str(resource_id), + body.model_dump(mode="json"), + request.state.request_id, + ) + return CanonicalJSONResponse(response, status_code=status) + + @router.get("/inspection-items/{item_id}/facts", operation_id="getCurrentFacts") + def get_current_facts(item_id: UUID, request: Request): + return read(request, "facts", item_id) + + @router.post( + "/inspection-items/{item_id}/facts", status_code=202, operation_id="factsInspectionItem" + ) + def edit_facts(item_id: UUID, body: FactsEditBody, request: Request): + return write(request, "facts", item_id, body) + + @router.get("/findings", operation_id="listFindings") + def list_findings(request: Request): + return read(request, "findings") + + @router.get("/findings/{finding_id}", operation_id="getFinding") + def get_finding(finding_id: UUID, request: Request): + return read(request, "finding", finding_id) + + @router.post("/findings/{finding_id}/confirm", operation_id="confirmFinding") + def confirm_finding(finding_id: UUID, body: VersionCommand, request: Request): + return write(request, "confirm", finding_id, body) + + @router.post("/findings/{finding_id}/reject", operation_id="rejectFinding") + def reject_finding(finding_id: UUID, body: VersionCommand, request: Request): + return write(request, "reject", finding_id, body) + + @router.post("/findings/{finding_id}/cannot-determine", operation_id="cannotdetermineFinding") + def cannot_determine(finding_id: UUID, body: FindingUncertainBody, request: Request): + return write(request, "cannot-determine", finding_id, body) + + app.include_router(router) diff --git a/apps/api/app/rules.py b/apps/api/app/rules.py new file mode 100644 index 0000000..0ae16bd --- /dev/null +++ b/apps/api/app/rules.py @@ -0,0 +1,125 @@ +"""Rule configuration and publication API adapters.""" + +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Request +from pydantic import Field + +from apps.api.app.foundations import Identifier +from apps.api.app.identity import Body, CanonicalJSONResponse, VersionCommand, pagination +from packages.application.rules import RuleApplication +from packages.domain.security import ServiceError +from packages.persistence.security import SESSION_COOKIE + + +class RuleSetCreate(Body): + name: Annotated[str, Field(min_length=1, max_length=200)] + description: Annotated[str, Field(min_length=1, max_length=2000)] + + +class RuleImportBody(Body): + bundle: dict + reason: Annotated[str, Field(min_length=1, max_length=2000, pattern=r"\S")] + + +class RuleSnapshotCreate(Body): + rule_version_ids: Annotated[list[Identifier], Field(min_length=1, max_length=100)] + + +def mount_rules(app, identity): + app.state.rules = RuleApplication(identity) if identity is not None else None + router = APIRouter(prefix="/api/v1") + + def context(request, allowed=()): + if any( + key not in allowed or len(request.query_params.getlist(key)) != 1 + for key in request.query_params + ): + raise ServiceError("VALIDATION_ERROR", 422, "Unexpected or duplicate query parameter") + if app.state.rules is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Rule service not configured") + return app.state.rules + + def read(request, kind, resource_id=None): + allowed = () if resource_id else ("page", "page_size", "laboratory_id") + service = context(request, allowed) + page, size = pagination(request) + laboratory_id = request.query_params.get("laboratory_id") + if laboratory_id is not None: + try: + laboratory_id = str(UUID(laboratory_id)) + except (ValueError, TypeError): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid laboratory_id") from None + return service.read( + kind, + request.cookies.get(SESSION_COOKIE), + request.state.request_id, + resource_id=str(resource_id) if resource_id else None, + laboratory_id=laboratory_id, + page=page, + page_size=size, + ) + + def write(request, kind, operation, body, *, resource_id=None): + service = context(request) + status, response = service.write( + kind, + operation, + request.cookies.get(SESSION_COOKIE), + request.headers["x-csrf-token"], + request.headers["idempotency-key"], + body.model_dump(mode="json"), + request.state.request_id, + resource_id=str(resource_id) if resource_id else None, + ) + return CanonicalJSONResponse(response, status_code=status) + + @router.post("/rule-sets", status_code=201, operation_id="createRuleSet") + def create_rule_set(body: RuleSetCreate, request: Request): + return write(request, "rule_set", "create", body) + + @router.get("/rule-sets", operation_id="listRuleSets") + def list_rule_sets(request: Request): + return read(request, "rule_set") + + @router.get("/rule-versions", operation_id="listRuleVersions") + def list_rule_versions(request: Request): + return read(request, "rule_version") + + @router.post("/rule-versions", status_code=201, operation_id="importRuleVersion") + def import_rule_version(body: RuleImportBody, request: Request): + return write(request, "rule_version", "import", body) + + @router.get("/rule-versions/{resource_id}", operation_id="getRuleVersion") + def get_rule_version(resource_id: UUID, request: Request): + return read(request, "rule_version", resource_id) + + @router.post( + "/rule-versions/{resource_id}/submit-approval", + operation_id="submitapprovalRuleVersion", + ) + def submit_rule_version(resource_id: UUID, body: VersionCommand, request: Request): + return write(request, "rule_version", "submit", body, resource_id=resource_id) + + @router.post("/rule-versions/{resource_id}/approve", operation_id="approveRuleVersion") + def approve_rule_version(resource_id: UUID, body: VersionCommand, request: Request): + return write(request, "rule_version", "approve", body, resource_id=resource_id) + + @router.post("/rule-versions/{resource_id}/publish", operation_id="publishRuleVersion") + def publish_rule_version(resource_id: UUID, body: VersionCommand, request: Request): + return write(request, "rule_version", "publish", body, resource_id=resource_id) + + @router.post("/rule-versions/{resource_id}/retire", operation_id="retireRuleVersion") + def retire_rule_version(resource_id: UUID, body: VersionCommand, request: Request): + return write(request, "rule_version", "retire", body, resource_id=resource_id) + + @router.get("/rule-bundles", operation_id="listRuleBundles") + def list_rule_bundles(request: Request): + return read(request, "rule_bundle") + + @router.post("/rule-bundles", status_code=201, operation_id="createRuleBundle") + def create_rule_bundle(body: RuleSnapshotCreate, request: Request): + return write(request, "rule_bundle", "create", body) + + app.include_router(router) diff --git a/apps/api/app/uploads.py b/apps/api/app/uploads.py index 23f1cea..54232b7 100644 --- a/apps/api/app/uploads.py +++ b/apps/api/app/uploads.py @@ -1,4 +1,4 @@ -"""Opt-in upload grants, validation acceptance, and read-only image metadata.""" +"""Opt-in upload grants, validation acceptance, image metadata and versioned downloads.""" from typing import Annotated, Literal from uuid import UUID @@ -8,6 +8,7 @@ from apps.api.app.foundations import Identifier from apps.api.app.identity import Body, CanonicalJSONResponse +from packages.application.image_download import ImageDownloadApplication from packages.application.uploads import UploadGrantApplication from packages.domain.security import ServiceError from packages.domain.uploads import MAX_BYTES @@ -30,6 +31,11 @@ class UploadComplete(Body): def mount_uploads(app, identity, storage): + app.state.image_downloads = ( + ImageDownloadApplication(identity, storage) + if identity is not None and storage is not None + else None + ) app.state.uploads = ( UploadGrantApplication(identity, storage) if identity is not None and storage is not None @@ -72,4 +78,25 @@ def image(image_id: UUID, request: Request): request.cookies.get(SESSION_COOKIE), str(image_id), request.state.request_id ) + @router.get("/images/{image_id}/download", operation_id="downloadImage") + def download( + image_id: UUID, request: Request, variant: Literal["analysis", "original"] = "analysis" + ): + if any( + key != "variant" or len(request.query_params.getlist(key)) != 1 + for key in request.query_params + ): + raise ServiceError("VALIDATION_ERROR", 422, "Unexpected or duplicate download query") + if app.state.image_downloads is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Image downloads not configured") + return CanonicalJSONResponse( + app.state.image_downloads.download( + request.cookies.get(SESSION_COOKIE), + str(image_id), + variant, + request.state.request_id, + ), + headers={"Referrer-Policy": "no-referrer"}, + ) + app.include_router(router) diff --git a/apps/worker/app/main.py b/apps/worker/app/main.py index 8f4abbf..228e05c 100644 --- a/apps/worker/app/main.py +++ b/apps/worker/app/main.py @@ -1,5 +1,6 @@ """Real Celery entrypoint; durable routing and execution belong to I-03.""" +import logging import os from typing import Any @@ -23,6 +24,41 @@ def create_celery_app() -> Celery: task_acks_late=True, task_reject_on_worker_lost=True, worker_prefetch_multiplier=1, + broker_transport_options={ + "visibility_timeout": 900, + "socket_connect_timeout": 2, + "socket_timeout": 5, + }, ) celery.task(name="labsafe.fixture.echo")(fixture_echo) + if ( + os.getenv("WORKER_IMAGE_VALIDATION_ENABLED", "0") == "1" + or os.getenv("WORKER_INFERENCE_ENABLED", "0") == "1" + or os.getenv("WORKER_RULE_EVALUATION_ENABLED", "0") == "1" + ): + from apps.worker.inference_pipeline import consume_inference + from apps.worker.rule_evaluation import consume_rule + + image_enabled = os.getenv("WORKER_IMAGE_VALIDATION_ENABLED", "0") == "1" + if image_enabled: + from apps.worker.image_validation import consume_image + from packages.storage.s3 import S3Settings + + S3Settings.from_environment(os.environ["PUBLIC_ORIGIN"], worker=True) + + @celery.task(name="labsafe.tasks.dispatch", ignore_result=True, shared=False) + def dispatch(message): + try: + if message.get("task_type") == "inference_pipeline": + consume_inference(message) + elif message.get("task_type") == "rule_evaluation": + consume_rule(message) + else: + if not image_enabled: + raise RuntimeError("validate_image consumer is disabled") + consume_image(message) + except Exception: + # DB is the retry/state authority. Do not leak message/provider/secret text. + logging.getLogger(__name__).warning("image_dispatch_failed") + return celery diff --git a/apps/worker/dispatch.py b/apps/worker/dispatch.py new file mode 100644 index 0000000..e93d955 --- /dev/null +++ b/apps/worker/dispatch.py @@ -0,0 +1,122 @@ +"""Opt-in development publisher/sweeper. No business task consumer is enabled.""" + +import argparse +import logging +import multiprocessing +import os +import time +from threading import Timer + +from apps.worker.app.main import create_celery_app +from packages.application.dispatch import DispatchPublisher, sweep_dispatch +from packages.application.inference_execution import InferenceExecution +from packages.application.job_execution import ImageExecution +from packages.application.rule_execution import RuleExecution +from packages.persistence.database import database_engine +from packages.shared.environment import development_environment + +PUBLISH_SECONDS = 20 +HEARTBEAT_SECONDS = 10 + + +def send_dispatch(event_id, message): + """Child has no DB connection; hard timeout can terminate a blocked broker call.""" + try: + app = create_celery_app() + app.send_task( + "labsafe.tasks.dispatch", + args=[message], + task_id=event_id, + queue="q.general", + serializer="json", + retry=False, + headers={"outbox_event_id": event_id}, + ) + app.close() + except Exception: + # Suppress tracebacks with broker credentials in a child process. + raise SystemExit(1) from None + + +class CeleryDispatchTransport: + def publish(self, event_id, message, renew): + child = multiprocessing.get_context("spawn").Process( + target=send_dispatch, + args=(event_id, message), + daemon=True, + ) + started = time.monotonic() + child.start() + + def terminate(): + # A slow DB heartbeat must not extend the broker child's wall budget. + try: + if child.is_alive(): + child.kill() + except ProcessLookupError: + pass + + deadline = Timer(max(0, PUBLISH_SECONDS - (time.monotonic() - started)), terminate) + deadline.daemon = True + deadline.start() + try: + child.join(max(0, HEARTBEAT_SECONDS - (time.monotonic() - started))) + if child.is_alive(): + renew() + child.join(max(0, PUBLISH_SECONDS - (time.monotonic() - started))) + if ( + child.is_alive() + or child.exitcode != 0 + or time.monotonic() - started >= PUBLISH_SECONDS + ): + raise RuntimeError("Broker publication unconfirmed") + finally: + deadline.cancel() + deadline.join() + if child.is_alive(): + child.kill() + child.join() + child.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("role", choices=("publisher", "sweeper")) + parser.add_argument("--once", action="store_true") + args = parser.parse_args() + development_environment() + if os.getenv("WORKER_DISPATCH_ENABLED", "0") != "1": + raise SystemExit("Set WORKER_DISPATCH_ENABLED=1 for development dispatch") + logging.basicConfig(level=logging.INFO) + engine = database_engine() + publisher = DispatchPublisher(engine, CeleryDispatchTransport()) + try: + while True: + try: + if args.role == "publisher": + count = publisher.publish_batch() + logging.info("dispatch_published=%d", count) + else: + expired = ImageExecution(engine).recover_expired() + expired += InferenceExecution(engine).recover_expired() + expired += RuleExecution(engine).recover_expired() + recovered, resent = sweep_dispatch(engine) + logging.info( + "execution_recovered=%d dispatch_recovered=%d redispatched=%d", + expired, + recovered, + resent, + ) + except Exception: + logging.error("dispatch_cycle_failed") + if args.once: + raise SystemExit(1) from None + if args.once: + break + time.sleep(1 if args.role == "publisher" else 15) + finally: + engine.dispose() + + +if __name__ == "__main__": + main() diff --git a/apps/worker/image_validation.py b/apps/worker/image_validation.py new file mode 100644 index 0000000..825909f --- /dev/null +++ b/apps/worker/image_validation.py @@ -0,0 +1,107 @@ +"""Bounded, cancellable preparation child; it receives no database or broker handle.""" + +import multiprocessing +import os +import time + +from packages.domain.image_validation import RejectedImage, ValidatedImage +from packages.domain.job_execution import LeaseLost +from packages.domain.security import ServiceError + +PREPARE_SECONDS = 120 + + +def prepare_child(channel, settings, tenant, input): + # Import in the child: real Pillow/storage work is isolated from lease heartbeat. + from packages.application.image_validation import prepare_image + from packages.storage.s3 import S3Storage + + storage = None + try: + storage = S3Storage(settings) + channel.send(("result", prepare_image(storage, tenant, input))) + except ServiceError as error: + channel.send(("error", error.code)) + except Exception: + channel.send(("error", "INTERNAL_ERROR")) + finally: + channel.close() + if storage is not None: + storage.close() + + +class BoundedImagePrepare: + def __init__(self, settings, tenant): + self.settings, self.tenant = settings, tenant + + def __call__(self, input, cancelled): + context = multiprocessing.get_context("spawn") + receive, send = context.Pipe(duplex=False) + child = context.Process( + target=prepare_child, args=(send, self.settings, self.tenant, input), daemon=True + ) + started = time.monotonic() + try: + child.start() + send.close() + while True: + if cancelled.is_set(): + raise LeaseLost() + remaining = PREPARE_SECONDS - (time.monotonic() - started) + if remaining <= 0: + raise ServiceError("STAGE_TIMEOUT", 503, "Image preparation timed out") + if receive.poll(min(0.1, remaining)): + try: + kind, value = receive.recv() + except EOFError: + raise ServiceError( + "INTERNAL_ERROR", 500, "Preparation child exited" + ) from None + if cancelled.is_set(): + raise LeaseLost() + if time.monotonic() - started >= PREPARE_SECONDS: + raise ServiceError("STAGE_TIMEOUT", 503, "Image preparation timed out") + if kind == "result" and isinstance(value, (ValidatedImage, RejectedImage)): + return value + code = ( + value + if kind == "error" + and value in {"DEPENDENCY_UNAVAILABLE", "STAGE_TIMEOUT", "INTERNAL_ERROR"} + else "INTERNAL_ERROR" + ) + raise ServiceError(code, 503, "Image preparation failed") + if not child.is_alive(): + raise ServiceError("INTERNAL_ERROR", 500, "Preparation child exited") + finally: + send.close() + receive.close() + if child.pid is not None: + if child.is_alive(): + child.kill() + child.join(timeout=2) + if child.is_alive(): + raise ServiceError("STAGE_TIMEOUT", 503, "Preparation child did not stop") + child.close() + + +def consume_image(message): + """DB failures leave the lease for sweeper; never pass provider errors to Celery logs.""" + from packages.application.job_execution import ImageExecution + from packages.persistence.database import database_engine + from packages.persistence.dispatch import validate_dispatch + from packages.persistence.image_validation import write_image_result + from packages.storage.s3 import S3Settings + + validate_dispatch(message) + settings = S3Settings.from_environment(os.environ["PUBLIC_ORIGIN"], worker=True) + engine = database_engine() + try: + return ImageExecution(engine).execute( + message, + BoundedImagePrepare(settings, message["tenant_id"]), + lambda connection, input, outcome: write_image_result( + connection, message["tenant_id"], message["task_id"], input, outcome + ), + ) + finally: + engine.dispose() diff --git a/apps/worker/inference_pipeline.py b/apps/worker/inference_pipeline.py new file mode 100644 index 0000000..1122339 --- /dev/null +++ b/apps/worker/inference_pipeline.py @@ -0,0 +1,15 @@ +"""Optional q.general consumer for inference_pipeline tasks.""" + +from packages.persistence.database import database_engine +from packages.persistence.dispatch import validate_dispatch + + +def consume_inference(message): + from packages.application.inference_execution import InferenceExecution + + validate_dispatch(message) + engine = database_engine() + try: + return InferenceExecution(engine).execute(message) + finally: + engine.dispose() diff --git a/apps/worker/rule_evaluation.py b/apps/worker/rule_evaluation.py new file mode 100644 index 0000000..88b82ef --- /dev/null +++ b/apps/worker/rule_evaluation.py @@ -0,0 +1,14 @@ +"""Optional q.general consumer for rule_evaluation tasks.""" + + +def consume_rule(message): + from packages.application.rule_execution import RuleExecution + from packages.persistence.database import database_engine + from packages.persistence.dispatch import validate_dispatch + + validate_dispatch(message) + engine = database_engine() + try: + return RuleExecution(engine).execute(message) + finally: + engine.dispose() diff --git a/apps/worker/run.py b/apps/worker/run.py index 7a71388..1119e5b 100644 --- a/apps/worker/run.py +++ b/apps/worker/run.py @@ -1,6 +1,7 @@ """Worker startup; --check validates configuration without connecting to Redis.""" import argparse +import os from .app.main import create_celery_app @@ -13,7 +14,20 @@ def main() -> None: if args.check: print("Worker configuration OK (broker connectivity not tested)") return - app.worker_main(["worker", "--loglevel=INFO"]) + general = any( + os.getenv(name, "0") == "1" + for name in ("WORKER_IMAGE_VALIDATION_ENABLED", "WORKER_INFERENCE_ENABLED") + + ("WORKER_RULE_EVALUATION_ENABLED",) + ) + app.worker_main( + [ + "worker", + "--loglevel=INFO", + "--pool=solo", + "--concurrency=1", + "--queues=q.general" if general else "--queues=celery", + ] + ) if __name__ == "__main__": diff --git a/docs/02-architecture/05-durable-jobs.md b/docs/02-architecture/05-durable-jobs.md index e5de073..9cfb717 100644 --- a/docs/02-architecture/05-durable-jobs.md +++ b/docs/02-architecture/05-durable-jobs.md @@ -64,3 +64,13 @@ replayJob 仅 admin，expected_version 匹配且 state=dead_letter/failed。先 ## 6. 必测故障点 提交前kill→无业务/无事件；提交后发布前kill→sweeper补发；发布后mark前kill→重复事件由inbox防重；AI完成后DB提交前kill→可重复计算但仅当前token提交；DB提交后ack前kill→重复消息无新增结果；lease过期后旧结果→拒绝；Redis清空→30秒扫描+发布恢复；超过4次→dead_letter；手工重放→新generation且完整历史。 + +## 7. 当前实现边界 + +I-03A1 接 validate_image TaskDispatch 的 publisher、发布租约回收与待执行任务补发，采用独立 READ COMMITTED 调度连接、逐条领取（每轮最多 100）和事务外有界发布；API 的 RR 快照不变。`last_dispatched_at` 为补发意图入库时间，不是 broker ack，见 [I-03A1 验收](../08-delivery/17-i03a1-durable-dispatch.md)。 + +I-03A2 补 validate_image 执行租约、attempt、独立心跳、围栏提交和技术失败/过期回收。锁 tenant 共享→owner→upload→image→task；技术失败不伪造 rejected。精确接入契约见 [I-03A2 验收](../08-delivery/18-i03a2-image-execution.md)。I-02F3 已接 opt-in general 消费、120 秒有界图像处理子进程及完整结果事务；默认 solo/concurrency=1，10 秒续租不依赖解码线程。内容拒绝不发缺少 analysis SHA 的 ImageValidated，成功事件写 Outbox，见 [I-02F3 验收](../08-delivery/19-i02f3-image-validation.md)。 + +I-03A3 接 getJob/listDeadLetters/replayJob 的 validate_image 分支。详情按图像实验室 READ；列表/重放仅 Admin，列表收 failed/dead_letter 两类技术终态。重放先短事务鉴权和加载固定输入，回滚临时幂等占位并释放锁，事务外 HEAD 原 staging VersionId，再短事务重验 Admin/归属/输入版本/task.version；generation、token、dispatch_sequence 增一，attempt 清零，原 payload/旧 attempts 保留，TaskDispatch/审计/202 幂等响应同事务。HEAD 只证明固定输入当时可读，SHA/解码仍交 F3；版本缺失 409，不回退最新版本。cached replay 也重验当前 Admin。精确故障证据见 [I-03A3 验收](../08-delivery/20-i03a3-image-job-replay.md)。其他领域事件发布/inbox、任务适配及其 replay 仍未接通；不能据此关闭整个 JOB-04 或业务闭环。 + +I-02G1 已将 `inference_pipeline` 按既有 task-message-v1 分支写入持久任务和 TaskDispatch outbox。I-03A2 现已补齐 opt-in 推理消费者：按 item→run→task 专属锁序领取、独立心跳、attempt/fencing、quality/runs 协议校验、`facts_ready` 首个 fact revision 围栏提交、规则任务入队及过期回收；详见[推理执行验收](../08-delivery/23-i03a2-inference-execution.md)。`rule_evaluation` 再按 item→fact/evaluation→task 锁序三值求值，原子生成 evaluation/findings 并收敛到人工复核，详见[规则评估验收](../08-delivery/24-i03a2-rule-evaluation.md)。rules 管理 API、report 下游、D-FINE-N/CUDA 真实执行、任务类型管理员 replay 仍须后续批次实现，不能将 fixture 视为生产模型。 diff --git a/docs/03-interfaces/01-api-contract.md b/docs/03-interfaces/01-api-contract.md index d79bb38..8e9e53d 100644 --- a/docs/03-interfaces/01-api-contract.md +++ b/docs/03-interfaces/01-api-contract.md @@ -86,18 +86,18 @@ CSV列固定：inspection_id,item_id,laboratory_id,location_id,run_id,fact_revis ### 5.4 巡检项查询与阶段能力（I-02E） - listInspectionItems 接受 page/page_size/laboratory_id，先授权父巡检再计数；getInspectionItem 不接受 query。父巡检不存在/不可见为 404；可见且无子项为 200 空页。筛选 lab 不存在/不可见为 404，同时可见但不匹配父巡检 lab 为 200 空页。 -- I-02E 接通这两个 GET 时累计 28 个公共业务 API；I-02F1/F2 新增上传授权/完成/图像查询后为 31 个。尚无 submit/retry/complete 巡检项写命令，因此 allowed_actions 固定为空能力集的投影，不能按状态补出前端按钮。 -- 领域动作探测直接复用 submit_item/retry_item/complete_item 守卫；未加载上下文与已核验空集合不同。启用动作必须同时具备真实命令事务、服务器完整可信上下文及同源回归；不是仅修改 capability。 +- I-02E 接通这两个 GET 时累计 28 个公共业务 API；I-02F1/F2 新增上传授权/完成/图像查询后为 31 个；I-02G1 新增 `submitInspectionItem` 后为 32 个。submit 已有真实事务入口，但查询端仍不推断默认图片选择，因此 `allowed_actions` 暂固定为空能力集的投影，不能按状态补出前端按钮。 +- 领域动作探测直接复用 submit_item/retry_item/complete_item 守卫；submit 的真实命令已由 I-02G1 接通，retry/complete 仍未接通。未加载上下文与已核验空集合不同。启用动作必须同时具备真实命令事务、服务器完整可信上下文及同源回归；不是仅修改 capability。 - 精确字段白名单、快照/权限和后续关闭条件见 [I-02E 验收](../08-delivery/14-i02e-item-query-api.md)。不修改公共 Schema、operationId 或生成源。 ### 5.5 上传授权与阶段能力（I-02F1） -- I-02F1 接通 POST /api/v1/uploads（createUpload，201）；API_UPLOADS_ENABLED 默认 0，依赖真实身份配置。未配置为 503，仅 dev/test。I-02F2 的 completeUpload/getImage 见 5.6；图片下载及解码/Worker 尚未接通，不能在获得 grant 或完成 PUT 后展示 image ready。 +- I-02F1 接通 POST /api/v1/uploads（createUpload，201）；API_UPLOADS_ENABLED 默认 0，依赖真实身份配置。未配置为 503，仅 dev/test。I-02F2 的 completeUpload/getImage 见 5.6；F3 的验证结果与 F4 下载见 5.7，不能在获得 grant 或完成 PUT 后展示 image ready。 - 请求严格为 owner_type、owner_id、filename、mime_type、size_bytes、sha256、captured_at，无 tenant_id/object_key/query；大小 1–15 MiB，JPEG/PNG/WebP，SHA 为 64 位小写十六进制。captured_at 为带时区 RFC3339，落库为 UTC 毫秒；不擅自添加新鲜度/未来窗口，不将客户端时间当可信证据。 - Item：当前 capture 权限及原创建者/admin，父巡检 draft/in_progress，子项 draft/uploaded/needs_retake/needs_review/failed。Task：当前受派人及 assignee 权限，in_progress/rejected；admin 不自动代替受派人，父巡检完成不阻断合法整改证据。 - 独立 10 次/分钟上传授权限流叠加普通写限流；每用户同时至多 10 个尚未过期的 granted，使用数据库锁保护。过期 grant 即使尚未被 sweeper 改状态也不占额度；Redis 不可用时拒绝而非绕过限流。 - grant 的 staging key 由服务端生成；PUT URL 同源 HTTPS:443、有效期 600 秒并签入 Content-Type/Host。相同 key/正文重放返回原响应/原到期时间，仍重验当前权限；过期 URL 不通过重放续签，新意图用新 key。 -- HEAD 已在 I-02F2 接到完成受理；固定版本 GET/SHA helper 仍未接到 Worker。真实桶版本化、IAM、反代大小限制和 MinIO 签名拒绝需另行实测，不能用 SDK/Stubber 成绩替代。授权实现、锁序、配置与历史证据见 [I-02F1 验收](../08-delivery/15-i02f1-upload-grant-api.md)。 +- HEAD 已在 I-02F2 接到完成受理；固定版本 GET/SHA 已由 F3 Worker 使用。真实桶版本化、IAM、反代大小限制和 MinIO 签名拒绝需另行实测，不能用 SDK/Stubber 成绩替代。授权实现、锁序、配置与历史证据见 [I-02F1 验收](../08-delivery/15-i02f1-upload-grant-api.md)。 ### 5.6 上传完成受理和图像元数据（I-02F2） @@ -106,7 +106,22 @@ CSV列固定：inspection_id,item_id,laboratory_id,location_id,run_id,fact_revis - 同事务写入 image=validating、upload=validating、validate_image task_runs=ready、TaskDispatch Outbox=pending、审计和幂等响应。无上传完成领域事件新枚举，不提前发 ImageValidated，不直接发 Redis。original_sha256 在 validating 阶段仍是待验证声明值。 - 同 key/正文重放返回原 202/原响应/request_id；新 key 命中已完成 upload 返回同一 image 当前投影，不新建/不再 HEAD。两者都重验当前 capture/assignee 权限和声明 hash，不因原 grant 过期或 owner 状态已推进而重复创建；已删除 image 的完成请求 404，不复活资源。 - getImage 只要求 owner.read，读权限不等于采集/受派权限；已删除图像可返回契约中的 deleted 元数据墓碑，但这不授予任何下载能力。内部 key/version、upload_id、tenant_id、legal_hold 和 task payload 均不公开。 -- 本批没有发布/领取/消费/图像 ready 提交；真实字节 SHA、解码、O/A 生成和固定版本缺失处理仍属于后续 Worker。实现、故障证据和依赖关闭清单见 [I-02F2 验收](../08-delivery/16-i02f2-upload-completion-api.md)。 +- F2 自身只受理；A1/A2 已接持久调度/执行，F3 已本地接真实字节 SHA、解码、O/A 生成和 ready/rejected 事务。历史实现与故障证据见 [I-02F2 验收](../08-delivery/16-i02f2-upload-completion-api.md)，F3 现行边界见下节。 + +### 5.7 图像验证结果与受控下载（I-02F3/F4） + +- F3 的 opt-in general Worker 只消费 validate_image；准确 GET、实算 SHA、真实 Pillow 解码、一次 EXIF 旋转、去元数据 RGB PNG 后登记 O/A 和 ready。内容拒绝登记 image/upload=rejected、task=succeeded；技术故障保留 validating、自动重试/技术终态。ImageValidated 只在成功时写 Outbox，下游发布/inbox 未接通。见 [F3 验收](../08-delivery/19-i02f3-image-validation.md)。 +- F4 接通 GET /api/v1/images/{id}/download（downloadImage），默认 variant=analysis，original 额外 Admin+原图读取审计。两者均按图像所属实验室 READ，只有 ready 且所选 O/A key/SHA/版本完整可签发，非 ready 返回 409。只允许单个 variant 参数，拒绝客户端对象 key/version/tenant 或额外 query。 +- DownloadGrant 仅 `{url,expires_at}`；同源 HTTPS SigV4 GET、固定数据库 VersionId、60 秒。签名原样使用，不跳到最新对象。API 返回 no-store/no-referrer，不重定向；原图审计与签发事务一起提交，审计失败不返回 grant。GET 不要求写命令头；每次原图签发独立审计，不做幂等缓存。 +- 当前身份/角色/session 和图片记录在事务内锁定，SDK 签名为本地操作，无 HEAD/GET 或凭据刷新。下载不改变 image/upload/task/event。签发不证明对象仍存在或真实 IAM/TLS 可用；对象 GET 失败不能据此更换 VersionId 或伪造成功。 +- 共用 120 次/分钟用户限流，Redis 故障不签发；不占写配额，不使用普通元数据读的降级。撤权后禁止新签发，既有 URL 到期前最长 60 秒仍可能有效。沿用 API_UPLOADS_ENABLED 和 API S3 secret；无新增迁移/配置。见 [F4 验收](../08-delivery/21-i02f4-image-download.md)。crop/报告下载与真实部署验收未完成。 + +### 5.8 提交巡检项与推理入队（I-02G1） + +- `POST /api/v1/inspection-items/{id}/submit` 接收 `ItemSubmit`，返回 `InferenceRun` 和 202。客户端只传 `expected_version` 及 1–3 个有序图片选择；服务端加载 item、父巡检、完整当前 findings、图片 metadata 和 laboratory activation。 +- 事务锁定 item 与完整图片/当前 findings，复用 `submit_item` guard。旧 run/evaluation/findings 只标记 superseded；新 run 固定已发布 model/dictionary/rule、pipeline、device、UTC reference date、图片 SHA/顺序与 input hash，并原子写入 `run_images`、`inference_pipeline` task、TaskDispatch outbox、审计和幂等响应。 +- 请求不能携带 tenant、laboratory、model、rule、device、date、object key/version 或 finding 列表。跨租户/实验室、非 ready 图片、版本过期、确认/派发/关闭 finding、activation 未发布均拒绝；任何写步骤失败整事务回滚。 +- 本端点只登记持久任务，不声明 AI 已执行或产生结果。独立推理 worker 的 lease、D-FINE-N/GPU 执行、超时、fencing、retry/replay 和结果提交属于后续 I-03/AI 批次；`allowed_actions` 在查询端仍不凭状态推断默认图片顺序。 ## 6. 命令实现映射和变更 diff --git a/docs/08-delivery/01-implementation-plan.md b/docs/08-delivery/01-implementation-plan.md index fb66cd4..984d01e 100644 --- a/docs/08-delivery/01-implementation-plan.md +++ b/docs/08-delivery/01-implementation-plan.md @@ -1,33 +1,39 @@ # 08.1 实施计划与首批任务 -状态（2026-10-01）：I-01A/B/C 已合并；I-02A/B/C/D/E/F1/F2 已通过本地相应验证，本次按用户授权提交特性分支并发起 PR。整个 I-02 尚未完成；业务闭环、真实模型和生产发布不得以局部成绩代替验收。阶段标准见[阶段门禁](05-stage-gates.md)，当前入口见[开发指南](../../DEVELOPMENT.md)。 +状态（2026-10-05）：I-01A/B/C 已合并；I-02A/B/C/D/E/F1/F2 已由 PR #9 合并。I-03A1 调度、I-03A2 上传验证/推理/规则执行基础、I-02F3 图像处理、I-03A3 图像任务查询/管理员重放、I-02F4 受控图片下载、规则管理和事实/人工复核代码已在本地完成；真实存储部署验收、remediation、report/export 和其余业务命令仍待关闭，完成情况见表及本批验收。整个 I-02/I-03 尚未完成；业务闭环、真实模型和生产发布不得以局部成绩代替验收。阶段标准见[阶段门禁](05-stage-gates.md)，当前入口见[开发指南](../../DEVELOPMENT.md)。 ## 1. 从“只有文档”开始 不要求用户先提供代码、训练后权重或运行报告。先创建受控开发环境和最小业务链路；真实模型/数据准备可以与业务代码并行，但任务仍按各自依赖验收。 -| 编号 | 任务与输入 | 必须交付 | 当前进度与证据 | 完成检查 / 失败处理 | -| ---------- | --------------------------------------- | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| I-01A | 编码基线；APP_ENV=dev；Python 3.11 独立业务/AI 环境 | API、Worker、AI、Web 骨架与进程入口、环境样例、依赖约束、CI | 已合并 PR #5；见[骨架验收](07-i01a-acceptance.md)，不是完整业务服务 | 干净环境 import/HTTP smoke/build；缺框架必须失败，不安装真实模型也可启动同协议 fixture | -| I-01B | 设计 DDL；专用可丢弃 MySQL | Alembic 初始迁移、bootstrap CLI、结构核对 | 已合并 PR #7；43 表、115 外键、迁移循环及约束验证，见[数据库验收](08-i01b-acceptance.md) | 空库 upgrade/verify；失败仅处理专用测试库，不触碰已有库 | -| I-01C | 数据约束和安全规范 | 基础仓储/租户事务、session/RBAC、幂等、安全原语测试 | 已合并 PR #8；见[安全原语验收](09-i01c-acceptance.md)；HTTP/并发端到端证据在 I-02 分批补齐 | DB-01/02、API-02/03、SEC-01/02 按实际覆盖记证；失败回滚，无旁路权限 | -| I-02 | 状态机与公共契约 | 巡检、上传、事实、人工复核、整改等命令/查询及认证管理 API | **进行中**；A/B/C/D/E/F1/F2 本地完成，累计注册 31 个公共业务 API；本次提交 PR，待远程 CI 和独立评审，完整业务 API 未完成 | 每个 guard 正/反例；锁定完整写集；相同请求重发无重复副作用；不能以身份/草稿/查询/上传接口验收整个 I-02 | -| I-02A | 领域状态机与守卫 | 13 个纯领域命令守卫、权限/版本/状态/证据测试 | 本地完成；542 个领域用例，见[领域守卫验收](10-i02a-domain-guards.md) | 后续应用事务加载可信完整聚合；纯守卫放行不等于数据库已提交 | -| I-02B | 认证和用户契约、安全原语 | 登录/会话/退出、用户列表/查询/创建/禁用，共 7 个 API；事务、限流、审计/幂等与故障验证 | 本地完成；597 项单元/边界、67 项持久化套件（有重叠），见[身份 API 验收](11-i02b-identity-api.md) | 同 key/版本并发、撤权后重放拒绝、最后管理员保护、失败原子回滚；仅 dev/test | -| I-02C | 角色契约与 I-02B 事务链路 | 角色列表/授予/撤销 3 个 API；精确 scope、version/epoch、撤会话、最后管理员及幂等保护 | 本地完成；612 项累计单元/边界、93 项持久化套件，其中新增 26 项角色集成测试；见[角色 API 验收](12-i02c-role-api.md) | 六角色 scope、跨租户/实验室拒绝、同 key/版本竞争、双管理员撤权、失败回滚与权限投影容量守卫 | -| I-02D | 组织、模板与巡检创建契约 | 学院/实验室/位置、模板创建/发布/克隆、巡检草稿 16 个 API；原子子项与作用域隔离 | 本地完成；691 项累计单元/边界、127 项持久化套件；本批新增 79 项单元/边界与 34 项数据库用例，见[基础 API 验收](13-i02d-foundation-api.md) | 父类型/active、发布和 family revision、≤100 项、同 key/多会话竞争、权限计数与全事务回滚；不包括上传/完成/取消 | -| I-02E | 巡检项查询契约与同源领域守卫 | 列表/详情 2 个 GET；白名单投影、授权后分页、动作探测基础与能力门禁 | 本地完成；961 项累计单元/边界、154 项持久化套件；新增 270 项单元/边界与 27 项数据库用例，见[巡检项查询验收](14-i02e-item-query-api.md) | 租户/实验室隔离、RR 快照、无业务副作用、未知上下文不放行；未接写命令，allowed_actions 暂为空 | -| I-02F1 | 上传授权契约、采集权限与同源 S3 规范 | createUpload、双额度守卫、原子 grant/审计/幂等；真实 SDK 本地签名及固定版本/SHA 适配基础 | 本地完成；1227 项累计单元/边界、178 项持久化套件（157 项真实 MySQL）；新增 266 项单元/边界与 24 项数据库用例，见[上传授权验收](15-i02f1-upload-grant-api.md)；默认关闭，仅 dev/test | 同用户并发不超额、重放不续签、撤权拒绝、失败全回滚；未实现 complete/解码/ready 或真实存储联调 | -| I-02F2 | F1 与持久任务/图像契约 | completeUpload/getImage、固定 staging version、原子 image+validate_image 任务/dispatch | 本地完成；1283 项主套件、213 项完整持久化回归（192 项真实 MySQL + 21 项重叠单元），本批新增 35 项数据库用例，见[上传完成验收](16-i02f2-upload-completion-api.md)；默认关闭 | 重复 complete 不重复建图/任务，外部 HEAD 在事务外、提交前重验；对接 general 消费、图像变换和真实存储验收后方可 ready | -| I-03 | 持久任务设计 | Outbox/inbox/publisher/sweeper、Worker、fixture-v1 AI 服务执行闭环 | 未完成；仅已有 I-01A Worker 入口与同协议 AI fixture，不具备持久任务闭环 | JOB-01/02/03 故障注入；断 Redis 后从 DB 恢复；不能用同步路由或 echo 任务替代 | -| I-ML-01 | ML-BASE-02 固定提交/配方；不依赖完整 API 或项目权重 | D-FINE 四类开发 checkpoint、映射/安全加载/qmax、ONNX、训练/CPU/CUDA lock 与 smoke | 未完成；当前仅有模型设计与合成 fixture，不是 D-FINE-N 实际执行 | 分类头未训必须 development；RGB/W/H/300 query/设备核对；不兼容形成 ADR | -| I-ML-02 | 数据来源授权与标注规范 | 授权台账、100 张标注试行、完整数据/split/审核 | 未完成；无本阶段真实数据验收证据 | 类别/scene 不泄漏；不足标待补充，无授权不抓取替代 | -| I-ML-03/04 | I-ML-01/02 与验收 policy | 四类训练/校准、训练后 50 图 PyTorch/ORT CPU/CUDA 一致性与资源报告 | 未开始验收；依赖真实数据、制品和固定环境 | AI-05 至 AI-09、MODEL-02；检测 GPU/OCR CPU；不以初始化模型或 mock 代替 | -| I-04 | 稳定 API 和 fixture 服务 | 前端采集/复核/整改/admin/report，开发环境标识 | 未完成；只有开发 Web 骨架，尚无完整登录、角色管理或业务 UI 联调 | UI-01、EXPORT-01、FLOW 闭环；旧轮询不能覆盖新 run | -| I-05 | 已实现各服务 | Compose、secret 挂载、探针、日志/指标、备份恢复 | 未完成；开发入口/局部探针不等于完整部署 | 隔离栈闭环、恢复验证；按服务/数据定位，不改证据掩盖失败 | -| R-01 | 机构批准 policy/规则、固定真实模型与 test | 真实业务评测、恢复/断公网演练、双人发布记录 | 未进入；production 仍禁止 | EVAL/OPS/Security 全部通过；缺批准或指标不足禁止发布 | - -状态口径：已合并表示对应历史 PR 已合并；本地完成不代表已提交、远程 CI 通过或独立审查批准。I-02A/B/C/D/E/F1/F2 为 I-02 的子批次，不改变总体验收标准。统计中的套件重叠范围见各批验收记录，不累加成唯一测试数；旧批成绩保留当时口径。F1 只开放上传授权；F2 接通完成受理和图像元数据，但没有 publisher/worker/ready 闭环。巡检项 allowed_actions 仍为空。 +| 编号 | 任务与输入 | 必须交付 | 当前进度与证据 | 完成检查 / 失败处理 | +| ---------- | --------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| I-01A | 编码基线；APP_ENV=dev；Python 3.11 独立业务/AI 环境 | API、Worker、AI、Web 骨架与进程入口、环境样例、依赖约束、CI | 已合并 PR #5；见[骨架验收](07-i01a-acceptance.md)，不是完整业务服务 | 干净环境 import/HTTP smoke/build；缺框架必须失败，不安装真实模型也可启动同协议 fixture | +| I-01B | 设计 DDL；专用可丢弃 MySQL | Alembic 初始迁移、bootstrap CLI、结构核对 | 已合并 PR #7；43 表、115 外键、迁移循环及约束验证，见[数据库验收](08-i01b-acceptance.md) | 空库 upgrade/verify；失败仅处理专用测试库，不触碰已有库 | +| I-01C | 数据约束和安全规范 | 基础仓储/租户事务、session/RBAC、幂等、安全原语测试 | 已合并 PR #8；见[安全原语验收](09-i01c-acceptance.md)；HTTP/并发端到端证据在 I-02 分批补齐 | DB-01/02、API-02/03、SEC-01/02 按实际覆盖记证；失败回滚，无旁路权限 | +| I-02 | 状态机与公共契约 | 巡检、上传、事实、人工复核、整改等命令/查询及认证管理 API | **进行中**；A/B/C/D/E/F1/F2 已合并 PR #9；含本地 A3 的 3 个 validate_image 接口、F4 下载、G1 submit、规则配置和事实/复核 API；本批业务/领域/协议 1477 项、完整持久化 MySQL 386 项通过；整改、report/export 和完整 UI 仍待完成 | 每个 guard 正/反例；锁定完整写集；相同请求重发无重复副作用；不能以身份/草稿/查询/上传接口验收整个 I-02 | +| I-02A | 领域状态机与守卫 | 13 个纯领域命令守卫、权限/版本/状态/证据测试 | 本地完成；542 个领域用例，见[领域守卫验收](10-i02a-domain-guards.md) | 后续应用事务加载可信完整聚合；纯守卫放行不等于数据库已提交 | +| I-02B | 认证和用户契约、安全原语 | 登录/会话/退出、用户列表/查询/创建/禁用，共 7 个 API；事务、限流、审计/幂等与故障验证 | 本地完成；597 项单元/边界、67 项持久化套件（有重叠），见[身份 API 验收](11-i02b-identity-api.md) | 同 key/版本并发、撤权后重放拒绝、最后管理员保护、失败原子回滚；仅 dev/test | +| I-02C | 角色契约与 I-02B 事务链路 | 角色列表/授予/撤销 3 个 API；精确 scope、version/epoch、撤会话、最后管理员及幂等保护 | 本地完成；612 项累计单元/边界、93 项持久化套件，其中新增 26 项角色集成测试；见[角色 API 验收](12-i02c-role-api.md) | 六角色 scope、跨租户/实验室拒绝、同 key/版本竞争、双管理员撤权、失败回滚与权限投影容量守卫 | +| I-02D | 组织、模板与巡检创建契约 | 学院/实验室/位置、模板创建/发布/克隆、巡检草稿 16 个 API；原子子项与作用域隔离 | 本地完成；691 项累计单元/边界、127 项持久化套件；本批新增 79 项单元/边界与 34 项数据库用例，见[基础 API 验收](13-i02d-foundation-api.md) | 父类型/active、发布和 family revision、≤100 项、同 key/多会话竞争、权限计数与全事务回滚；不包括上传/完成/取消 | +| I-02E | 巡检项查询契约与同源领域守卫 | 列表/详情 2 个 GET；白名单投影、授权后分页、动作探测基础与能力门禁 | 本地完成；961 项累计单元/边界、154 项持久化套件；新增 270 项单元/边界与 27 项数据库用例，见[巡检项查询验收](14-i02e-item-query-api.md) | 租户/实验室隔离、RR 快照、无业务副作用、未知上下文不放行；未接写命令，allowed_actions 暂为空 | +| I-02F1 | 上传授权契约、采集权限与同源 S3 规范 | createUpload、双额度守卫、原子 grant/审计/幂等；真实 SDK 本地签名及固定版本/SHA 适配基础 | 本地完成；1227 项累计单元/边界、178 项持久化套件（157 项真实 MySQL）；新增 266 项单元/边界与 24 项数据库用例，见[上传授权验收](15-i02f1-upload-grant-api.md)；默认关闭，仅 dev/test | 同用户并发不超额、重放不续签、撤权拒绝、失败全回滚；未实现 complete/解码/ready 或真实存储联调 | +| I-02F2 | F1 与持久任务/图像契约 | completeUpload/getImage、固定 staging version、原子 image+validate_image 任务/dispatch | 本地完成；1283 项主套件、213 项完整持久化回归（192 项真实 MySQL + 21 项重叠单元），本批新增 35 项数据库用例，见[上传完成验收](16-i02f2-upload-completion-api.md)；默认关闭 | 重复 complete 不重复建图/任务，外部 HEAD 在事务外、提交前重验；对接 general 消费、图像变换和真实存储验收后方可 ready | +| I-03 | 持久任务设计 | Outbox/inbox/publisher/sweeper、Worker、fixture-v1 AI 服务执行闭环 | **进行中**；A1/A2/F3/A3 已接 validate_image 调度/执行/图像消费者/人工重放；仍缺领域事件发布、其他任务领域适配及其 replay、inbox/AI 闭环 | JOB-01/02/03/04 故障注入；断 Redis 后从 DB 恢复；不能用同步路由或 echo 任务替代 | +| I-03A1 | F2 的 validate_image TaskDispatch | 发布租约、事务外有界发布、过期发布回收、从 DB 补发至 q.general | 本地完成；1290 项业务/领域/安全、239 项完整持久化（218 MySQL + 21 单元）通过；新增 28 + 26 项，见[持久调度验收](17-i03a1-durable-dispatch.md)；默认关闭，未提交/Push/PR | SKIP LOCKED 并发、过期 owner 拒绝、插入失败回滚、真实 Redis 丢消息恢复；不标图像 ready | +| I-03A2 | A1 与领域任务入口 | 执行 lease/attempt/heartbeat/fencing、失败收敛、任务回收与 replay 基础 | **部分实现**；validate_image、inference_pipeline 与 rule_evaluation 已接专属执行基础；推理见[推理执行验收](23-i03a2-inference-execution.md)，规则见[规则评估验收](24-i03a2-rule-evaluation.md)；report 仍待接 | 聚合先于 task 加锁，旧租约/取消拒绝、4 次耗尽；report 任务与完整 JOB-04 仍待实现 | +| I-02F3 | F2、A1 与 A2 图像执行基础 | general 验证消费、固定版本/SHA、图像变换、O/A 与完整 ready/rejected 事务 | **本地代码完成**；1360 项主套件 + 4 协议 + 21 单元、318 项完整持久化（297 MySQL + 21 重叠单元）通过；新增 49 + 37 项，见[图像验证验收](19-i02f3-image-validation.md)；默认关闭，未提交/Push/PR；真实存储验收待 | 真实 Pillow、准确对象版本、同 SHA 复用、原子状态/审计/事件与旧 attempt 拒绝已测；UP-03/SEC-03 部署验收仍待 | +| I-03A3 | A2/F3 与 Job 公共契约 | validate_image 任务详情、技术终态列表、管理员固定输入重放；generation/fence/dispatch/审计/幂等同事务 | **本地代码完成**；1429 项主套件/协议/单元、357 项完整持久化（336 MySQL + 21 重叠单元）通过；新增 44 + 39 项，见[图像任务重放验收](20-i03a3-image-job-replay.md)；未提交/Push/PR | HEAD 在事务外；重验 Admin/归属/输入版本；旧 generation/lease 拒绝；历史保留；其他类型未开放 | +| I-02F4 | F3 ready 图像与 downloadImage 契约 | 分析图 READ、原图 Admin+审计；规范 O/A 固定版本同源 60 秒 GET 签名 | **本地代码完成**；1469 项主套件/协议/单元、383 项完整持久化（362 MySQL + 21 重叠单元）通过；新增 40 + 26 项，见[受控图片下载验收](21-i02f4-image-download.md)；未提交/Push/PR | 独立 HMAC 核签；锁内当前鉴权；非 ready/坏引用不签名；审计失败不返回 grant；Redis 故障不降级；真实 IAM/TLS 待验收 | +| I-02G1 | submitInspectionItem 与推理任务入队 | 可信 item/image/activation 锁定；旧 run/finding supersede；新 run、run_images、inference_pipeline task、TaskDispatch、审计、幂等同事务 | **本地代码完成**；新增 API/持久化验收与 task-message-v1 校验；既有 383 项真实 MySQL 回归通过，本批隔离 MySQL 专测 1 项通过，见[提交入队验收](22-i02g1-submit-inference-enqueue.md) | activation 只取已发布 model/dictionary；图片清单仅客户端选择，模型/规则/设备/日期服务端冻结；规则评估由后续专属任务接续；真实生产闭环仍未完成 | +| I-ML-01 | ML-BASE-02 固定提交/配方；不依赖完整 API 或项目权重 | D-FINE 四类开发 checkpoint、映射/安全加载/qmax、ONNX、训练/CPU/CUDA lock 与 smoke | 未完成；当前仅有模型设计与合成 fixture，不是 D-FINE-N 实际执行 | 分类头未训必须 development；RGB/W/H/300 query/设备核对；不兼容形成 ADR | +| I-ML-02 | 数据来源授权与标注规范 | 授权台账、100 张标注试行、完整数据/split/审核 | 未完成；无本阶段真实数据验收证据 | 类别/scene 不泄漏；不足标待补充，无授权不抓取替代 | +| I-ML-03/04 | I-ML-01/02 与验收 policy | 四类训练/校准、训练后 50 图 PyTorch/ORT CPU/CUDA 一致性与资源报告 | 未开始验收；依赖真实数据、制品和固定环境 | AI-05 至 AI-09、MODEL-02；检测 GPU/OCR CPU；不以初始化模型或 mock 代替 | +| I-04 | 稳定 API 和 fixture 服务 | 前端采集/复核/整改/admin/report，开发环境标识 | 未完成；只有开发 Web 骨架，尚无完整登录、角色管理或业务 UI 联调 | UI-01、EXPORT-01、FLOW 闭环；旧轮询不能覆盖新 run | +| I-05 | 已实现各服务 | Compose、secret 挂载、探针、日志/指标、备份恢复 | 未完成；开发入口/局部探针不等于完整部署 | 隔离栈闭环、恢复验证；按服务/数据定位，不改证据掩盖失败 | +| R-01 | 机构批准 policy/规则、固定真实模型与 test | 真实业务评测、恢复/断公网演练、双人发布记录 | 未进入；production 仍禁止 | EVAL/OPS/Security 全部通过；缺批准或指标不足禁止发布 | + +状态口径：已合并表示对应历史 PR 已合并；本地完成不代表已提交、远程 CI 通过或独立审查批准。I-02A/B/C/D/E/F1/F2 为 I-02 的子批次，表中各子批的本地成绩为历史验收记录，现均已随 #9 合并。统计中的套件重叠范围见各批验收记录，不累加成唯一测试数。F2 接受上传并登记元数据，A1/A2 接调度与执行，F3 已本地接通验证消费和 ready/rejected 事务，A3 补图像技术失败后的人工重放，F4 补 ready 图像受控下载签发，G1 接通提交入口但不声称推理已执行；真实存储链路、推理执行和结果闭环仍待验收。巡检项 allowed_actions 仍为空，直到查询端具备可信的默认图片选择上下文。 ## 2. 数据库验证的精确任务边界 @@ -49,4 +55,4 @@ IRR六项设计前置按[关闭清单](06-implementation-readiness.md)执行。 ## 5. 协作和Push -设计基线与 I-01A/B/C 已通过各自 PR 合并。I-02A/B/C/D/E/F1/F2 本次按用户授权从 `wsq/i-02a-auth-api` 提交 PR；不直接推送 main，不自动合并或替代独立审查。各批验收中的未提交状态是当时的历史记录；当前交付状态以本节及 GitHub PR 为准。远程 CI 和数据库/权限变更的协作者审查仍为合并前置。 +设计基线与 I-01A/B/C 已通过各自 PR 合并；I-02A/B/C/D/E/F1/F2 已由 PR #9 合并。当前 I-03A1/A2、I-02F3、I-03A3 和 I-02F4 位于特性分支 `wsq/i-03a-durable-dispatch`；工作区保留各批改动，尚未提交/Push/PR。本轮没有重新 fetch 或核验远程合并状态。不直接推送 main，不自动合并或替代独立审查。各批验收中的未提交状态是当时的历史记录；当前交付状态以本节及 GitHub PR 为准。远程 CI 和数据库/权限变更的协作者审查仍为合并前置。 diff --git a/docs/08-delivery/17-i03a1-durable-dispatch.md b/docs/08-delivery/17-i03a1-durable-dispatch.md new file mode 100644 index 0000000..6497c66 --- /dev/null +++ b/docs/08-delivery/17-i03a1-durable-dispatch.md @@ -0,0 +1,81 @@ +# 08.17 I-03A1：上传验证任务的持久调度 + +状态（2026-10-01）：基于已合并 PR #9 的 `origin/main`（070ed74），在 `wsq/i-03a-durable-dispatch` 本地完成并通过第 4 节验证；尚未提交或推送，不引用旧 PR 的 CI 作为本批证据。 + +## 1. 交付范围 + +I-03A 拆为调度与执行两批。本批 I-03A1 接通已存在的 `validate_image` 初始 TaskDispatch，交付 MySQL Outbox → Celery/Redis `q.general` 发布、发布租约回收和 ready/retry_wait 任务补发。公共 API 仍为 31 个，无新增依赖、表或迁移。 + +| 入口 | 本批行为 | 保留给后续 | +|---|---|---| +| publisher | 领取 Outbox、事务外发布、确认后登记 published | 其他 task_type 和领域事件路由 | +| sweeper | 回收过期 Outbox 租约；为到期待执行任务原子登记新 dispatch | 执行租约回收、attempt 重试与死信 | +| Worker | 配置 JSON、late ack、visibility_timeout=900；发布专用子进程 | `labsafe.tasks.dispatch` 消费处理器、领域结果事务 | + +图像继续为 validating；task 为 ready 不是图像 ready。本批不实现 event_inbox、图像解码/真实 SHA/O/A 派生、人工 replay、推理或生产部署，不关闭整个 JOB-01/02/03/04。 + +## 2. 数据与锁语义 + +### 2.1 发布 + +`packages/persistence/dispatch.py` 只操作调度记录；`packages/application/dispatch.py` 负责事务边界；`apps/worker/dispatch.py` 提供两个独立进程入口。 + +1. 调度连接使用 **READ COMMITTED**，避免扫描索引更新时 RR 间隙锁阻塞；不改变 API 的 RR 一致性查询。所有时间条件和租约均来自 MySQL UTC_TIMESTAMP(3)。 +2. 每轮最多处理 100 条，每次只领取 1 条到期 pending 的 validate_image TaskDispatch：`FOR UPDATE SKIP LOCKED`，owner 为新 UUID，lease=30 秒，attempts+1，提交事务。逐条领取避免批尾等待网络时租约耗尽。 +3. 严格检查消息版本 1.1、字段白名单、UUID、trace、计数、带时区时间、payload 与 resource_id，以及外层租户/aggregate_id 一致性。暂未实现的 task_type/领域事件保持原记录，不交给 fixture。 +4. 退出数据库事务后启动 spawn 发布子进程，发送 JSON Celery task `labsafe.tasks.dispatch` 至 `q.general`。Celery task_id 与 `outbox_event_id` header 均为稳定 Outbox.id；同事件重投保持该 ID，但 Celery ID 不被当作去重保证。 +5. 发布预算 20 秒，10 秒仍在执行时用独立连接续约 30 秒；独立定时器到期终止发布子进程，不因数据库续约阻塞而放宽 broker 执行预算。退出路径 kill/join/close；主进程失败后由租约恢复，不能假设取消意味着 broker 未收消息。 +6. 子进程确认发送后，重新锁行，再取 DB now，检查 owner/state/未过期 lease 后登记 published。旧 owner、过期 owner 或被接管 owner 不能续租、确认或退回 pending。网络/协议失败保留记录并延后 30 秒重试，仅记录 event_id，不输出异常正文、payload 或 Redis URL。 + +### 2.2 补发 + +每轮 sweeper 先用短事务回收最多 100 条过期发布租约，再用另一事务扫描最多 100 条到期 `ready/retry_wait` 的 validate_image 任务。`last_dispatched_at` 为空或距今至少 30 秒时，锁 task 后原子完成 sequence+1、更新时间、登记新 TaskDispatch。 + +`last_dispatched_at` 表示最近一次补发意图入库时间，不是 broker ack；F2 的初始值 NULL 允许首次 sweeper 立即补发。其后 30 秒内不会再次补发。原事件即使 published 也不阻止补发，因此 Redis 丢消息可以恢复。payload、generation 和资源 ID 取 MySQL，不取队列中的业务参数；重新调度创建新 event_id/trace_id，原有历史不修改。 + +终态、执行中或尚未到 available_at 的任务不补发。新 Outbox 插入失败时 sequence/version/last_dispatched_at 一起回滚。本批不改变 task attempt、fencing_token、业务 owner/image 状态；后续消费者必须按 generation/sequence/领域状态重新查库和领取，不能直接执行消息 payload。 + +## 3. 开发运行与回退 + +仅在专用 dev/test 数据库与可丢弃 Redis 上启用，默认 `WORKER_DISPATCH_ENABLED=0`，production 启动拒绝。数据库 secret/命名限制沿用开发指南 3.5/3.6，REDIS_URL 沿用业务环境；不要放入 Git 或共享日志。 + +~~~powershell +$env:APP_ENV = 'dev' +$env:WORKER_DISPATCH_ENABLED = '1' +# 沿用已配置的 DATABASE_URL_FILE、DATABASE_SCHEMA、REDIS_URL +.\.venv-business\Scripts\python.exe -m apps.worker.dispatch publisher --once +.\.venv-business\Scripts\python.exe -m apps.worker.dispatch sweeper --once +~~~ + +去掉 `--once` 后，在两个独立终端分别运行；publisher 每轮结束等待 1 秒，sweeper 等待 15 秒。日志输出发布、回收和补发数量；本批没有完整指标/告警平台。坏记录不删除，发布失败日志需人工调查；无效 task payload 会回滚该轮补发，须先修复可信数据来源后重试，不静默跳过坏数据。 + +尚无业务消费者，不要将当前 fixture Worker 手动订阅到 `q.general`，否则 Celery 会丢弃未注册任务。仅作调度验证，避免长期空转造成消息与 Outbox 积压。此限制不会因 published 状态消失，不能用该入口处理实际业务图片。 + +回退：停止两个调度进程并恢复开关为 0；保留 task/outbox/审计历史，不降级数据库、不清空 Redis 或业务表。正在发布的消息可能重复，后续处理器仍须提供持久围栏。 + +## 4. 验证记录 + +环境：Windows、Python 3.11.4、MySQL 8.0.33；数据库由仓库隔离启动器创建临时 datadir/随机端口/随机 schema，Redis 由测试夹具启动独立子进程，不使用现有业务服务。迁移 head=0001_initial，43 表/115 外键核对通过。 + +| 验证命令（业务 Python 环境） | 本轮结果 | +|---|---| +| `python -m pytest tests/business tests/domain tests/security -q` | 1290 passed，含本批 28 项；1 项既有 Starlette/AnyIO 弃用警告 | +| `python -m pytest tests/persistence/test_unit.py tests/protocol -q` | 25 passed（21 持久化单元 + 4 协议） | +| `PYTEST_ADDOPTS='-x -k dispatch_mysql'` 后运行隔离 MySQL 启动器 | 新增 26 项全部通过，213 deselected；包含真实 Redis/Celery 发送/丢消息/补发 | +| 完整 `python tools/database/run_mysql_tests.py --mysqld ` | 239 passed，1 既有 Starlette/AnyIO 弃用警告；MySQL 8.0.33、43 表/115 外键 | +| `python -m ruff check apps packages tests tools/database` | PASS | +| `python -m ruff format --check apps packages tests tools/database` | 111 文件通过 | +| `python -B tools/design/build_specs.py --check` | 16 生成物无漂移 | +| `python -B tools/design/test_readiness_design.py` | 148 项合成检查通过 | +| `python -B tools/design/validate_specs.py` | PASS，231 文档链接 | + +本批新增 28 项业务测试和 26 项真实 MySQL 调度测试。完整持久化 239 项由 218 项真实 MySQL 用例和 21 项单元用例组成；后者与单独运行的 25 项套件重叠，不累加成唯一测试数。首次集成发现测试夹具缺租户 FK、RR 扫描间隙锁和 Redis 空队列断言差异，修正后完整回归通过。真实存储、业务消费者、独立 AI 和生产验收不包含在这些结果中。 + +故障覆盖：事务外发布、SKIP LOCKED 双 publisher、过期/错 owner/接管后回写拒绝、发布后未登记的重复发送、失败退避与日志脱敏、并发 sweeper 单次补发、插入失败原子回滚、未来/终态任务排除、真实 Redis 消息删除后从 published 记录恢复。 + +## 5. 下一批关闭清单 + +1. I-03A2：任务领取、attempt/执行租约/heartbeat/fencing；锁领域聚合后锁 task，当前版本与消息 generation/sequence 全部重验；旧执行者不能提交。 +2. 同一领域收敛函数处理主动失败和过期执行租约，完成重试/4 次耗尽/永久错误/人工 replay，覆盖 JOB-04；不能只更新 task 状态。 +3. 领域事件消费接入 event_inbox 和原子业务事务；扩展任务类型前补对应路由、入口与失败守卫。 +4. I-02F3 接 general 图像验证、固定版本 GET/真实 SHA、解码和 O/A 生成，再原子登记 ready/rejected/ImageValidated。后续执行真实 IAM/存储/HTTPS 联调，关闭 SEC-03/UP-03。 diff --git a/docs/08-delivery/18-i03a2-image-execution.md b/docs/08-delivery/18-i03a2-image-execution.md new file mode 100644 index 0000000..609f255 --- /dev/null +++ b/docs/08-delivery/18-i03a2-image-execution.md @@ -0,0 +1,97 @@ +# 08.18 I-03A2：上传验证任务执行基础 + +状态（2026-10-01）：继续在 `wsq/i-03a-durable-dispatch` 上实现，保留上一批未提交改动。本批范围为 validate_image 的执行租约、围栏、技术失败重试与过期回收；验证见第 4 节。尚未提交/Push/PR，I-03A2 的其他任务类型及人工 replay 集成仍未完成。 + +## 1. 交付与边界 + +| 层 | 文件 | 责任 | +|---|---|---| +| 领域值 | `packages/domain/job_execution.py` | 不可变 ImageInput/ImageLease、租约异常、技术错误白名单、5/30/120 秒加 0–20% 抖动 | +| 持久化 | `packages/persistence/job_execution.py` | 锁聚合后锁 task、领取/attempt、续租、围栏提交、失败/回收原子写入 | +| 应用 | `packages/application/job_execution.py` | 独立短事务、prepare 外部工作、后台心跳、结果事务、限量回收 | +| 进程 | `apps/worker/dispatch.py` | 既有 sweeper 增加过期执行租约回收，再执行 A1 发布回收/补发 | + +无新增公共 API、迁移、依赖或配置项。没有注册 `labsafe.tasks.dispatch` Celery 消费任务，fixture Worker 仍不得订阅 q.general；仅补齐未来真实验证处理器可调用的服务。测试中的 rejected handler 是合成事务夹具，不能冒充 SHA/解码/图像 ready 能力。 + +## 2. 开发者接入契约 + +### 2.1 消息与领取 + +调用 `ImageExecution(engine).execute(message, prepare, write_result)`；实例仅在 APP_ENV=dev/test 下创建。API 与普通 Worker 启动不自动创建或领取任务。message 必须符合已实现的 validate_image 消息 1.1 分支；不接受其他 task_type。 + +查库只用 tenant_id/task_id。数据库记录决定资源、payload、attempt、generation 和输入；消息不会覆盖它们。旧 generation 直接不执行；generation 相同才比较 sequence，旧 sequence 不执行，超前 generation/sequence 拒绝。不存在、终态、执行中、未到 available_at 均不重复执行。资源和 payload 不一致拒绝，不将消息字段传给存储操作。 + +事务使用独立 READ COMMITTED 连接，锁序如下： + +1. tenant 共享锁；非 active 租户不能开始/继续执行。 +2. locator 普通读定位资源，不先锁 task。 +3. 巡检图片锁 inspection → inspection_item；整改图片锁 remediation_task。 +4. 锁 upload → image → task_run，重新核对引用、租户/实验室、状态与 task payload。 + +巡检 owner 仅允许 draft/uploaded/needs_retake/needs_review/failed，父巡检必须 draft/in_progress；整改仅允许 in_progress/rejected。upload/image 均须 validating；key 必须是登记的 staging key，声明 SHA/MIME/大小一致，精确 object_version 不得空或 `null`。grant 到期不影响已经受理的任务，Worker 不重新签发 grant。 + +领取成功在同事务将 attempt+1、fencing_token+1、state=leased、owner=新 UUID、lease=DB now+60 秒，并插入唯一 running task_attempt。插入失败整体回滚。ImageLease 保存 DB 输入、upload/image version 与租约身份；不携带队列提供的对象路径。此阶段不改 image/owner 业务状态。 + +### 2.2 外部工作、心跳与结果事务 + +`prepare(input, cancelled)` 在无数据库事务状态执行，只收到不可变 ImageInput 和取消 Event。后续处理器必须执行固定版本 GET/SHA/解码，遵循有界 I/O，并检查取消信号；本批没有外部图像工作硬超时或强制终止实现，不应接入无界/阻塞不响应取消的处理器。 + +独立 heartbeat 线程每 10 秒另开数据库连接，按同一锁序检查 owner/token/generation/attempt、未过期 lease、owner 当前可采集状态、固定输入和 upload/image version，成功延长 60 秒。可变 owner 按当前状态重验，不能因另张图片将 draft 推进 uploaded 而拒绝合法输入；upload/image 本身在同一 attempt 内必须保持原版本。 + +续租返回 false 或数据库异常均锁定本次 attempt 的失败标记，发出取消信号；即使随后数据库恢复，`execute` 也不再调用 commit。线程无法在退出时 2 秒内收敛也禁止提交，等待数据库操作结束及租约回收。心跳日志只记 task_id，不输出驱动异常、URL 或输入。`LeaseHeartbeat` 是外部工作包围器，不在结果事务内等待心跳线程。 + +`write_result(connection, input, outcome)` 是服务端受信任的事务回调，不能来自 HTTP/队列。进入前重新锁聚合和 task，核 owner/token/generation/attempt/未过期 lease 及原输入版本。回调负责 image/upload、owner、审计和应有事件的**完整**写集，不得做对象/网络调用。 + +回调返回后要求 image 为 ready/rejected 且 upload 状态相同，再以新 DB now 重验租约；成功才将 attempt/task 同事务标 succeeded，清空 lease。内容不合格而正确登记 rejected 也是“验证任务成功完成”，不是技术失败。回调抛错、未落终态、提交前租约到期、attempt 更新失败都会整体回滚。重复消息不会再生成结果。 + +该基础只检查终态与租约，不替代 F3 对 ready 对象证据、owner 迁移、审计和 ImageValidated 的完整校验。不能传入只改 status 的业务实现；当前唯一此类 handler 位于测试夹具中。 + +### 2.3 技术失败与过期回收 + +目前 `fail` 只接受契约中的 DEPENDENCY_UNAVAILABLE、STAGE_TIMEOUT（可重试）和 INTERNAL_ERROR（不可重试）。IMAGE_INVALID/HASH_MISMATCH 等必须由验证处理器形成拒绝结果，不能借该 helper 跳过业务状态事务。 + +| 情况 | task | attempt | 图像/owner | +|---|---|---|---| +| 当前租约技术可重试失败，attempt 1–3 | retry_wait，available_at=DB now+退避 | failed | 保留 validating/现有 owner | +| 第 4 次可重试失败 | dead_letter | failed | 保留输入，等待人工处理 | +| INTERNAL_ERROR | failed | failed | 保留输入，不伪造图像无效 | +| 已过期租约，attempt 1–3 / 第 4 次 | retry_wait / dead_letter | abandoned | 保留输入 | +| owner 已取消/终结/不可采集或输入无效 | failed/LEASE_LOST | 已领取 attempt 标 failed；回收时 abandoned | 不复活或覆盖当前业务状态 | +| 旧 owner/token/generation 或普通失败请求已过期 | 不变 | 不变 | 不变 | + +失败和回收共用收敛函数，同事务更新 task/attempt 并 token+1；回收者不依赖旧 Worker。sweeper 每轮最多发现 100 个过期任务，再逐个按领域优先锁序重验 token/generation/过期条件，避免反向先锁 task；两个 sweeper 竞争时仅一个成功。回收后沿用 A1 的到期补发，Redis 仍不是状态源。 + +技术失败不生成 ImageValidated：该事件契约要求有效 analysis_sha256，不能为网络失败伪造字段。没有人工 replay API，也不自动重置 generation/attempt；管理员后续需要通过带权限、版本、输入可用性重验、审计和幂等的命令恢复。 + +## 3. 运行与回退 + +沿用开发指南 3.11、WORKER_DISPATCH_ENABLED 和数据库/Redis 配置，无新增启用命令。sweeper 日志增加 `execution_recovered`；先回收执行租约，再执行发布租约回收与补发。尚无验证消费者，默认情况下无新增 execution lease 是预期。 + +回退停止调度进程、关闭开关并恢复上一版程序；保留 task/attempt/outbox 及业务数据。任何遗留 leased 等待后续正确版本的 sweeper 回收，不能清表、手动伪造成功或将失效 lease 续上。生产仍禁止。 + +## 4. 验证 + +新增 21 项单元测试、42 项真实 MySQL 测试。数据库为隔离启动器创建的 MySQL 8.0.33 临时 schema，head=0001_initial，43 表/115 外键无漂移；未使用已有业务数据库。 + +| 实际命令（业务 Python 3.11.4） | 本轮结果 | +|---|---| +| `python -m pytest tests/business tests/domain tests/security -q` | 1311 passed，含新增 21 项；1 项既有 Starlette/AnyIO 弃用警告 | +| `python -m pytest tests/persistence/test_unit.py tests/protocol -q` | 25 passed（21 持久化单元 + 4 协议） | +| `PYTEST_ADDOPTS='-x -k job_execution_mysql'` 后运行隔离 MySQL 启动器 | 42 passed，239 deselected | +| 完整 `python tools/database/run_mysql_tests.py --mysqld ` | 281 passed，1 既有 Starlette/AnyIO 弃用警告；MySQL 8.0.33、43 表/115 外键 | +| `python -m ruff check apps packages tests tools/database` | PASS | +| `python -m ruff format --check apps packages tests tools/database` | 116 文件通过 | +| `python -B tools/design/build_specs.py --check` | 16 生成物无漂移 | +| `python -B tools/design/test_readiness_design.py` | 148 项合成检查通过 | +| `python -B tools/design/validate_specs.py` | PASS，234 文档链接 | + +各套件分开计数，持久化单元用例与完整持久化套件重叠，不重复累加。整改 owner 的首轮失败来自通用 Graph 夹具预先添加的 run_images 引用；改为尚未被 run 选用的采集阶段夹具后，42 项数据库测试全部通过。另补了心跳取消导致外部异常时禁止旧执行者写失败状态的单元回归。 + +覆盖重复领取、消息版本/身份/未来和过期调度、attempt 插入回滚、旧租约续租/失败/提交拒绝、固定对象版本变化、取消和 busy owner、整改 owner、双 sweeper、两种四次耗尽、迟到结果、回调/attempt 失败回滚、事务外工作及独立心跳失效。迁移和采集接口回归仍须完整运行；模拟 handler 不算真实图像或模型验收。 + +## 5. 后续关闭清单 + +1. I-02F3：实现有界的固定版本下载/实算 SHA/解码/方向归一/EXIF 清理/O/A 派生，将完整结果写集接入本批 `execute`，再注册 general 消费任务并验证 late ack/reject-on-loss。当前不应开启 consumer。 +2. I-03A2 剩余：按 inference/rule/report 各自领域入口接入专属失败收敛，覆盖 JOB-04；不能把本批 image guard 直接用于其他任务。 +3. 人工 replayJob：权限、幂等、expected_version、制品/精确输入仍存在、generation 与调度序号递增、审计及旧 attempt 历史；当前不可通过改数据库标 ready 替代。 +4. event_inbox/领域事件消费、真实存储 IAM/HTTPS、独立 AI 与最终业务闭环仍按各自阶段验收。 diff --git a/docs/08-delivery/19-i02f3-image-validation.md b/docs/08-delivery/19-i02f3-image-validation.md new file mode 100644 index 0000000..2a3ed8a --- /dev/null +++ b/docs/08-delivery/19-i02f3-image-validation.md @@ -0,0 +1,97 @@ +# I-02F3 图像验证处理与原子结果 + +日期：2026-10-02。范围：在 I-02F2 上传受理、I-03A1 调度及 I-03A2 执行围栏上接通真实图像处理。代码仅供显式启用的 dev/test 环境；生产发布、真实 MinIO/IAM/HTTPS 验收和完整业务闭环仍未完成。本批没有新增公共 API、数据库迁移或模型推理能力，也未提交、Push 或创建 PR。 + +## 1. 实现入口与职责 + +| 文件 | 职责 | +|---|---| +| [image_validation.py](../../packages/domain/image_validation.py)（domain） | 成功/拒绝结果类型、内容错误白名单、O/A 规范 key、ready 证据约束 | +| [image_validation.py](../../packages/application/image_validation.py)（application） | 固定版本读取后的完整解码、归一化、事务外对象持久化；不接 DB connection | +| [s3.py](../../packages/storage/s3.py) | 准确版本 GET、字节 SHA、同 SHA 复用检查、固定源版本 COPY、分析图 PUT | +| [image_validation.py](../../packages/persistence/image_validation.py)（persistence） | 在持有领域锁/围栏的事务中登记图像、上传、归属、审计和成功事件 | +| [job_execution.py](../../packages/persistence/job_execution.py) | 提交前后核租约；ready 必须有合法 O/A key/version/SHA/尺寸；task/attempt 同事务终结 | +| [image_validation.py](../../apps/worker/image_validation.py)（worker） | 有界 spawn 处理子进程及私有消费入口；子进程不接 DB/broker handle | +| [main.py](../../apps/worker/app/main.py)、[run.py](../../apps/worker/run.py) | 默认关闭的 Celery 注册、q.general/solo/concurrency=1 启动入口、安全错误日志 | + +启动、secret 配置、回退步骤见 [开发指南 3.12](../../DEVELOPMENT.md)。API/Worker/AI 均使用 Python 3.11；Pillow 11.3.0 加入业务 requirements/worker extra，不向独立 AI 环境添加 DB/S3/Redis 依赖。 + +## 2. 执行顺序 + +1. 严格校验 schema 1.1 的 validate_image 消息；消息只定位任务，输入来自 DB。其他 task_type、未来 generation/sequence 和额外字段不执行。 +2. 通过 A2 短事务领取 lease，锁 tenant 共享→owner（巡检父/项或整改任务）→upload→image→task。固定 staging key/version、期望 SHA、字节大小/MIME、image/upload version。 +3. 事务结束后启动独立心跳与处理子进程。子进程总预算 120 秒（含启动、GET、解码、O/A 检查与写入），父进程每 100ms 检查取消/预算；心跳每 10 秒用独立连接续租 60 秒。失败一经锁定不会因后续续租恢复而接受结果。 +4. GET 必须带固定 VersionId，并复核响应版本、大小、MIME；流式读块 64KiB、最多 15 MiB，关闭 Body。实算 SHA 与声明值比较；不信 ETag/HEAD 自报 SHA，也不回退到最新 staging 版本。 +5. 真实 Pillow verify 后重新打开并完整 load。仅 JPEG/PNG/WebP，真实格式必须与声明 MIME 一致；拒绝多帧动画、不可解码/截断、各边超过 10000 或总像素超过 40M。EXIF transpose 一次，转 RGB，使用新像素对象输出 PNG，去除 GPS/EXIF/ICC/text。分析图不缩放；检测/质量阶段的 resize 属于后续任务。 +6. 用已校验 ID/SHA 生成规范 O/A key。已有对象先 HEAD 固定其准确版本，再 GET 实算 SHA；相同字节复用版本，异 hash/尺寸/MIME 冲突报 INTERNAL_ERROR，不覆盖。不存在才创建；原图 COPY 来源准确 staging 版本，A PUT 为 RGB PNG。写入必须返回有效非 null VersionId。分析 PNG 上限 128 MiB，容纳 40M RGB 像素的编码开销。 +7. 子进程只返回结果证据（key/version/SHA/尺寸）或内容拒绝码，不返回原始图片字节。失租/超时终止并 join 子进程；失租抛 LEASE_LOST，超时进入 A2 技术失败恢复。 +8. 停止准备阶段心跳后进入 A2 围栏提交。重检 DB 时间、owner/输入版本、owner 状态、lease owner/token/generation/attempt。结果、审计、成功 Outbox 与 task/attempt 终态在同一事务；任何一步失败全部回滚。commit 最后再次核 DB 时间，不让等待锁耗尽租约的执行者提交。 + +对象存储不参与 DB 原子事务。已写 O/A 但登记失败会留下孤儿；下一 attempt 实算同 SHA 后可复用。并发创建相同内容可能产生多个对象版本，只有当前围栏登记的准确版本成为证据。未登记版本留待受控清理，不授予 normal Worker 删除/列桶权限。 + +## 3. 结果与失败语义 + +| 情况 | 图像/上传与归属 | task/attempt | 审计/事件 | +|---|---|---|---| +| 成功 | ready，登记 O/A 准确 key/version、analysis SHA、尺寸；原始 SHA 保持已验证声明值 | 同事务 succeeded | service actor=NULL 的 image.validate；ImageValidated schema 1.1 | +| 内容失败 | rejected，清空分析证据；原图保持已固定 staging 引用以供历史定位 | succeeded，表示验证处理完成 | 审计记录错误码；不伪造缺少 analysis SHA 的 ImageValidated | +| 对象存储不可达/限流/无权；超时 | 保持 validating | A2 的 retry_wait，最多 4 次后 dead_letter | 本批不产生图像结果事件 | +| 内部错误/受控对象内容冲突 | 保持 validating | failed/INTERNAL_ERROR | 不覆盖冲突对象；不误判上传原图内容不合格 | +| 失租/owner 取消或终结/输入变化 | 拒绝结果登记；已有 O/A 仍为孤儿 | 由 A2 围栏/回收收敛 | 无新增图像结果、审计或事件 | +| DB 提交前失败/提交后重复消息 | 前者全回滚；后者 no-op | 当前记录保持唯一 | 不增加重复事件或审计 | + +内容错误白名单：OBJECT_NOT_FOUND（固定 staging 版本不存在）、IMAGE_INVALID、IMAGE_TOO_LARGE、UNSUPPORTED_MEDIA_TYPE、HASH_MISMATCH。既有 O/A 读取过程的版本消失属于存储依赖故障，不以其取代 staging 或覆盖对象。 + +成功时 draft item→uploaded，其他允许采集态保持；inspection draft→in_progress。拒绝时按现行状态机，有其他 ready 图则 item=uploaded，否则 draft。整改图像只在 task in_progress/rejected 时接纳，验证不替代证据提交命令，task 状态保持。 + +ImageValidated 暂存 pending Outbox；A1 publisher 仅发布 TaskDispatch。领域事件 publisher/inbox/下游消费者未实现，不以事件已入库声称通知或推理已经执行。消费者捕获异常时只记录固定错误日志，DB 是重试/状态真相源；malformed 消息不会触发图像处理，不将 provider 原文、URL、图片或凭据交给 Celery 错误日志。 + +## 4. 本地验证证据 + +业务/图像用例见 [test_image_validation.py](../../tests/business/test_image_validation.py)，真实数据库用例见 [test_image_validation_mysql.py](../../tests/persistence/test_image_validation_mysql.py)。全部图片在内存中生成，不含真实实验室数据。 + +本批新增 49 项业务用例、37 项真实 MySQL 用例。不得把 skip、Stubber 或 synthetic storage version 当作真实存储部署证据。 + +| 验证 | 2026-10-02 本地结果 | +|---|---| +| 业务/领域/安全/协议/持久化单元 | 1385 passed = 1360 主套件 + 4 协议 + 21 持久化单元；1 项既有 Starlette/AnyIO 弃用警告 | +| I-02F3 真实 MySQL 定向回归 | 37 passed；无跳过；包含 Redis/Celery solo Worker | +| 完整持久化回归 | 318 passed = 297 项真实 MySQL + 21 项单元；1 项既有弃用警告，862.11s；21 项单元与上行重叠，不能相加宣称互不重叠 | +| 静态/格式/依赖 | Ruff check PASS；format 122 files PASS；pip check PASS | +| 契约/设计/文档 | build_specs --check 16 制品未漂移；148 synthetic readiness；validate_specs PASS、248 文档链接 | +| 独立 AI 环境 | 15 项 fixture + 4 项协议 PASS；业务/数据库/S3/模型依赖隔离 PASS | + +本地通过与远程 CI、独立审批分开记录；本轮未创建 PR，不沿用 #9 的 CI 成绩。 + +- 真实 Pillow：JPEG/PNG/WebP、灰度/调色板/16-bit/带 alpha PNG、EXIF 一次旋转、元数据剥离、动画拒绝、格式伪装/截断及尺寸边界。 +- 真实 SDK Stubber：准确 GET/CopySource VersionId、精确写入参数与版本、既有对象实算 SHA 复用、错误 metadata 不欺骗复用、冲突/存储错误无覆盖。 +- 真实 spawn：Pillow 处理结果跨进程返回，卡住的子进程超时/取消后 kill+join，无遗留子进程；存储版本在该用例中为 synthetic。 +- 真实 MySQL 8.0.33：完整 ready/rejected、巡检/整改归属、合法事件 schema、服务审计、六个提交步骤失败回滚、失租/取消/输入版本变化、禁止裸 ready、并发重复领取只写一次。 +- 真实 Redis/Celery solo Worker：消费 TaskDispatch 并提交 rejected 结果；图像准备使用 synthetic handler，不证明真实 MinIO 链路。另有真实解码→MySQL ready、对象适配 Stubber 分层证据。 + +### 可复现命令 + +~~~powershell +.\.venv-business\Scripts\python.exe -m pytest tests/business tests/domain tests/security tests/protocol tests/persistence/test_unit.py -q +.\.venv-business\Scripts\python.exe tools/database/run_mysql_tests.py --mysqld '<专用 MySQL 8.x mysqld 路径>' +.\.venv-business\Scripts\python.exe -m ruff check apps packages tests tools/database +.\.venv-business\Scripts\python.exe -m ruff format --check apps packages tests tools/database +.\.venv-business\Scripts\python.exe -m pip check +python -B tools/design/build_specs.py --check +python -B tools/design/test_readiness_design.py +python -B tools/design/validate_specs.py +~~~ + +数据库启动器创建新临时目录、随机 loopback 端口和专用 schema，仅清理自己创建的实例。需要 `LABSAFE_TEST_REDIS_SERVER` 指向 Redis 可执行文件；CI 沿用已配置的可丢弃服务。 + +## 5. 尚待关闭的部署验收 + +| 关闭项 | 执行动作与通过条件 | +|---|---| +| UP-03/SEC-03 真实对象链路 | 专用 versioned MinIO + nginx 同源 HTTPS；用 API grant PUT→complete 202→publisher/general/sweeper→GET image ready，用 SDK GET 登记的准确 O/A 版本比对实算 SHA、方向与元数据 | +| 凭据隔离 | API 凭据不能 PUT O/A；general 凭据不能 Delete/ListBucket/管理、跨授权前缀访问；浏览器不能任意 GET O/A；记录实际拒绝结果 | +| 版本与故障 | staging 重复 PUT 后只处理 pinned version；缺失版本 rejected；写 O 后断存储保持 validating；恢复重试同 SHA 复用；DB 登记前 kill 后旧结果拒绝 | +| 生命周期/孤儿 | S 当前/非当前版本 24h 清理实测；业务 O/A 不受该策略影响。受控孤儿清理按精确版本核 DB 引用，独立 cleanup 凭据和任务尚待实现 | +| 生产资源/发布 | 40M 高熵图的实际耗时、峰值内存和 120s 预算测量；容器资源、进程关闭、凭据范围和完整系统门禁验收后才允许 production | + +以上是实现后的验收事项，不重新打开模型选型或重做领域设计。当前关闭本地代码批次，继续后续提交/推理业务入口和 I-03 其他任务适配；生产和真实存储门禁保持未完成。 diff --git a/docs/08-delivery/20-i03a3-image-job-replay.md b/docs/08-delivery/20-i03a3-image-job-replay.md new file mode 100644 index 0000000..6ba99bc --- /dev/null +++ b/docs/08-delivery/20-i03a3-image-job-replay.md @@ -0,0 +1,97 @@ +# I-03A3 图像任务查询与管理员重放 + +日期：2026-10-02。范围：在 A1/A2/F3 已实现的 validate_image 上接通 3 个既有公共契约；只供 dev/test。没有新增迁移、模型能力或配置项。其他任务类型、领域事件 publisher/inbox、真实存储部署与生产验收仍未完成。本批保留在 `wsq/i-03a-durable-dispatch`，未提交、Push 或创建 PR。 + +## 1. 接口与权限 + +| 方法与路径（前缀 /api/v1） | 契约 operationId | 权限与行为 | +|---|---|---| +| GET /jobs/{id} | getJob | 图像实验室 READ；仅 validate_image，其他类型/不可见资源 404 | +| GET /dead-letters | listDeadLetters | safety_admin；当前租户 validate_image 的 failed、dead_letter 技术终态；page/page_size/laboratory_id | +| POST /dead-letters/{id}/replay | replayJob | safety_admin；VersionCommand；202 返回 ready Job，异步执行 | + +列表包括两类可人工重放的技术终态：failed 为永久技术错误，dead_letter 为自动重试耗尽。并非所有列出的任务都仍可重放：归属取消、输入状态变化或固定 staging 版本过期会在命令中返回 409。内容拒绝的任务为 succeeded，不进入此列表；修正图片后新建上传。 + +Job 投影严格使用公共字段：id/created_at/updated_at/version/task_type/resource_id/state/attempt/replay_generation/available_at/last_error_code。没有 tenant_id、payload、对象 key/version、lease owner、fencing token、原始错误或凭据。时间统一 UTC。列表总数与页面使用一个 RR 快照，created_at DESC、id DESC 稳定排序。 + +查询只需要身份服务配置；首次重放还需要 API S3 配置。HTTP 写入沿用同源 Origin、JSON、CSRF、Idempotency-Key；拒绝未知/重复 query 和非契约 body。expected_version 必须取 Job.version，不是 Image.version。配置与示例见 [开发指南 3.13](../../DEVELOPMENT.md)。 + +## 2. 实现入口 + +| 文件 | 职责 | +|---|---| +| [jobs.py](../../apps/api/app/jobs.py)（API） | UUID、查询参数、VersionCommand 和 202 HTTP 适配 | +| [jobs.py](../../packages/application/jobs.py)（application） | preflight/Redis 限流、两阶段重放、事务重验、幂等响应 | +| [job_replay.py](../../packages/domain/job_replay.py) | 严格命令、Admin、租户/类型/版本/终态/输入与计数容量守卫 | +| [jobs.py](../../packages/persistence/jobs.py)（persistence） | 白名单查询与 RR 分页；task/dispatch/审计原子写入 | +| [job_execution.py](../../packages/persistence/job_execution.py) | 与 claim/commit/recover 共用领域先锁的固定输入加载 | +| [s3.py](../../packages/storage/s3.py) | inspect_pinned_staging：HEAD 原 VersionId；metadata/错误校验 | + +复用 A2 的可信 ImageInput，不允许客户端传 payload、generation、对象版本或替代输入。数据库原 payload/resource/logical_key 保持，准确对象 VersionId 从已登记 image 读取。 + +## 3. 重放执行顺序 + +1. 严格校验 body 的 expected_version/reason。短 preflight 检查当前身份、CSRF、Admin；Redis 写限流在 DB 事务外，失败返回 503，不能访问对象或修改任务。 +2. 第一个短事务 locate session scope，锁 tenant 共享，建立临时幂等占位，再重新认证当前身份/Admin。已完成的相同 key/body 重验当前可见性后直接返回原始 202 body/request_id/version，不重新 HEAD、不检查已过时的任务终态或 expected_version。相同 key、不同 body 返回 409。 +3. 新命令按 tenant→user/roles/session→owner→upload→image→task 加锁。巡检锁父再锁项；整改锁整改任务。验证 task.type=validate_image、版本匹配、state=failed/dead_letter；image/upload 仍 validating、归属允许采集且未取消/终结、lab/key/SHA/MIME/固定版本与 payload 一致。generation/dispatch_sequence/version 的 INT 容量不足返回 409。 +4. 复制 ImageInput 后抛内部控制异常，使整个准备事务回滚，临时幂等占位和锁均释放。没有提交长期 pending，也没有持锁等待存储。 +5. 事务外 HEAD 原 staging key+VersionId；校验返回版本、大小、MIME、DeleteMarker。没有 latest fallback、重新 grant、签名续期或生命周期延长。缺失固定版本 409 STATE_CONFLICT；存储访问故障/无权 503 DEPENDENCY_UNAVAILABLE；metadata 不匹配 409。HEAD 只验证当时可用，不证明实际 SHA 或解码成功。 +6. 第二个短事务重复幂等、身份、CSRF、Admin、task.version/状态和领域输入检查。当前 ImageInput 必须等于准备阶段副本（含 image/upload version），否则 409。HEAD 期间撤权/会话撤销、owner 取消或输入变化不会提交；数据库死锁最多重新执行 3 次短事务，不重复存储 I/O。 +7. 原子更新 task：generation+1、fencing_token+1、dispatch_sequence+1、attempt=0、state=ready、version+1；清 lease/heartbeat/started/finished/error，available_at=DB now、last_dispatched_at=now。last_dispatched_at 表示补发意图登记时间，不代表 broker ack。 +8. 同事务写新的 schema 1.1 TaskDispatch、操作者 job.replay 审计、完整 202 幂等响应。任一步失败全部回滚。旧 attempts/原 payload/图片/上传不改；publisher/general Worker/sweeper 沿用 A1/A2/F3，新 generation 从 attempt=1 开始。 + +同 key 的并发请求允许收到缓存 202 或短暂 409 REQUEST_IN_PROGRESS；不同 key 使用同旧 version，最多一个提交，其余 409。已受理 202 不是执行成功证明，GET Job/Image 查询后续状态。HEAD 后原版本仍可能消失，由 F3 准确 GET、SHA/解码和围栏收敛，不能跨存储与 MySQL 声称原子保证。 + +## 4. 重放与失败处理 + +| 情况 | 结果与后续动作 | +|---|---| +| 非 Admin/实验室不可见/跨租户 | 403/404；会话失效 401；不访问对象、不创建 generation | +| Job.version 过时 | 409 VERSION_CONFLICT；重新 GET，确认是否仍需要重放，使用新 key | +| ready/retry_wait/leased/succeeded | 409 STATE_CONFLICT；不要用人工重放干预仍在运行的任务 | +| owner 已取消/终结或 image/upload 非 validating | 409；不复活领域状态；按业务入口处理 | +| 固定 staging 版本不存在 | 409；重新上传；24h 生命周期仍适用 | +| S3 未配置/不可达/无权或 Redis 失败 | 503；恢复依赖后同 key/body 可再次尝试，无遗留 pending | +| 结果写入步骤异常 | task/dispatch/audit/idempotency 全回滚；内部异常脱敏 500 INTERNAL_ERROR，DB 不可用 503 DEPENDENCY_UNAVAILABLE | +| 原 generation 消息、旧执行 lease | no-op/LEASE_LOST；无旧结果写回；新任务历史不被旧执行者覆盖 | +| 相同 key/body 已成功 | 原始响应；即使 Worker 已推进任务也不重复执行命令；当前身份/Admin 仍须有效 | + +人工重放与自动重试区分：一次 generation 内仍最多 4 个 attempts；Admin 可在技术终态手动开启下一代，需要 reason 与审计。此接口不重建 inference run、替换 facts、广播通知或实现其他任务 replay。 + +## 5. 本地验证证据 + +业务边界见 [test_job_api.py](../../tests/business/test_job_api.py)，数据库/HTTP/Redis 见 [test_job_api_mysql.py](../../tests/persistence/test_job_api_mysql.py)。本批新增 44 项业务用例、39 项真实 MySQL 用例；对象 HEAD 使用真实 SDK Stubber，未连接真实 MinIO。 + +| 验证 | 2026-10-02 本地结果 | +|---|---| +| I-03A3 定向真实 MySQL | 39 passed，259.58s；无跳过；MySQL 8.0.33、43 表、115 外键、head=0001_initial | +| 业务/领域/安全/协议/持久化单元 | 1429 passed = 1404 主套件 + 4 协议 + 21 持久化单元，31.41s；1 项既有 Starlette/AnyIO 弃用警告 | +| 完整持久化回归 | 357 passed = 336 项真实 MySQL + 21 项单元，1084.41s；无跳过；1 项既有弃用警告；21 项单元与上行重叠 | +| 静态/格式/依赖 | Ruff check PASS；128 files format PASS；pip check PASS | +| 契约/设计/文档 | build_specs --check 16 制品未漂移；148 synthetic readiness；validate_specs PASS、262 文档链接 | + +覆盖：Job/JobPage schema、全部状态投影、tenant/lab/角色隔离、Admin 两阶段重验、撤权后缓存拒绝、精确版本 HEAD/无 latest、HEAD 外无 task/image 锁与 pending、输入/owner 变化拒绝、四处写入失败全回滚、同/不同 key 并发只一代、原 attempts 保留、旧消息/lease 拒绝，新 generation 实际提交 F3 内容拒绝结果、无存储配置的重试与缓存响应、RR count/items 同快照。 + +### 可复现命令 + +~~~powershell +.\.venv-business\Scripts\python.exe -m pytest tests/business tests/domain tests/security tests/protocol tests/persistence/test_unit.py -q +$env:LABSAFE_TEST_REDIS_SERVER = '<专用 Redis 可执行文件路径>' +.\.venv-business\Scripts\python.exe tools/database/run_mysql_tests.py --mysqld '<专用 MySQL 8.x mysqld 路径>' +.\.venv-business\Scripts\python.exe -m ruff check apps packages tests tools/database +.\.venv-business\Scripts\python.exe -m ruff format --check apps packages tests tools/database +.\.venv-business\Scripts\python.exe -m pip check +python -B tools/design/build_specs.py --check +python -B tools/design/test_readiness_design.py +python -B tools/design/validate_specs.py +~~~ + +定向可临时设置 PYTEST_ADDOPTS='-x -k job_api'；完整回归必须移除该过滤变量。启动器只创建/清理自己的临时 MySQL/Redis，不触碰已有数据库或服务。完整持久化套件含 21 项与主套件重叠的单元，不相加宣称互不重叠。 + +本地代码与验证批次已关闭；远程 CI 和独立审批没有执行。AI 环境、模型和前端本批未修改，不以业务/数据库成绩替代其独立验收。 + +## 6. 回退与后续关闭项 + +停止 API 暂停新人工重放；已受理任务由 Worker 持续处理。如需暂停整条链路，按开发指南先停领取、预算内排空，再停 publisher/sweeper/general；保留任务、审计和全部历史，没有删除或降级迁移。 + +UP-03/SEC-03 仍需 versioned MinIO、真实 IAM、同源 HTTPS、实际 S3 生命周期与异常恢复验证，见 [F3 部署验收清单](19-i02f3-image-validation.md)。本批 Stubber 不能替代这些证据。其他类型的领域守卫/重放、事件 publisher/inbox、提交与 AI 闭环继续按实施计划完成；整个 I-02/I-03/JOB-04 未关闭，production 仍禁止。 diff --git a/docs/08-delivery/21-i02f4-image-download.md b/docs/08-delivery/21-i02f4-image-download.md new file mode 100644 index 0000000..9cbbc15 --- /dev/null +++ b/docs/08-delivery/21-i02f4-image-download.md @@ -0,0 +1,94 @@ +# I-02F4 受控图片下载与原图审计 + +日期：2026-10-02。范围：基于 F3 ready 图像，接通既有 downloadImage 契约的 analysis/original 分支，仅 dev/test。新增 1 个 API，累计注册 35 个公共业务 API。无新迁移、配置、前端或模型能力。位于 `wsq/i-03a-durable-dispatch`，本轮未提交、Push、创建 PR 或核验远程状态。 + +## 1. 接口与失败语义 + +GET `/api/v1/images/{id}/download`；默认 analysis，唯一 query 为单个 variant=analysis/original。UUID 规范化，拒绝重复/未知 query、客户端 tenant/key/version。返回现有 DownloadGrantResponse：`{data:{url,expires_at},request_id}`；不重定向，不返回原始对象字节。 + +| 情况 | 行为 | +|---|---| +| ready analysis | 当前图像实验室 READ；固定 A key/version，60 秒 GET 签名 | +| ready original | 先 READ，再额外 Admin；固定 O key/version，签发与敏感读取审计同事务 | +| 未认证/会话失效 | 401；不进入 SDK 签名 | +| 跨 tenant/lab、对象不存在 | 404；不暴露状态、key、version | +| 可见资源但缺少 Admin | original 为 403，analysis 不受此额外权限限制 | +| validating/rejected/deleted | 409 STATE_CONFLICT；无 staging GET、latest 回退或“失败原图”旁路 | +| 所选 O/A key/SHA/version 不完整或不规范 | 409 STATE_CONFLICT；不签任意 key | +| 签名配置/生成异常、无有效期限 | 503 DEPENDENCY_UNAVAILABLE，错误脱敏 | +| Redis 故障/额度耗尽 | 503/429；不使用普通 GET 的保守读降级；不签发 | +| 原图审计或 DB 提交失败 | 不返回 grant；事务回滚，内部异常脱敏 500，DB 不可用 503 | + +共用 API_IDENTITY_ENABLED/API_UPLOADS_ENABLED、API S3 凭据和同源 HTTPS 配置；未配置为 503。GET 使用 session，不要求写命令的 CSRF/Idempotency-Key；每次原图签发独立审计，没有 grant 缓存。配置/使用/回退见 [开发指南 3.14](../../DEVELOPMENT.md)。 + +## 2. 实现入口 + +| 文件 | 职责 | +|---|---| +| [uploads.py](../../apps/api/app/uploads.py) | downloadImage 路由、strict query、no-store/no-referrer 响应 | +| [image_download.py](../../packages/application/image_download.py)（application） | preflight、事务外限流、事务内当前鉴权/签名/原图审计 | +| [image_download.py](../../packages/domain/image_download.py)（domain） | READ/Admin、ready、规范 O/A key/SHA/准确版本与不可变引用 | +| [images.py](../../packages/persistence/images.py) | 图像 FOR SHARE current read；不依赖旧 RR 快照签发 | +| [s3.py](../../packages/storage/s3.py) | 固定显式凭据本地 presign_image_get；精确 URL 范围与 TTL 复核 | +| [rate_limit.py](../../packages/application/rate_limit.py) | download_grant，共用 120/min 用户配额，Redis 失败不降级，不占写配额 | + +没有向 AI 环境引入 S3、数据库或业务授权依赖。Worker 凭据保持独立，下载仅用 API secret，其允许范围见 [IRR-04 操作矩阵](../06-security/01-security-privacy.md)。 + +## 3. 顺序与事务边界 + +1. HTTP 验 UUID/variant/query，应用 preflight 检查身份；download_grant 在数据库事务外用 Redis 检查共享用户配额。不是信任 preflight 的旧权限。 +2. 开始短 RR 事务。authenticate 持有 tenant→user/roles→session 当前锁；重新检查会话、epoch、用户/tenant 状态和权限。按 tenant+image_id 对 image FOR SHARE 作当前读，直到提交都阻止图片状态/版本/对象引用被并发修改。下载不锁 owner 聚合，不修改领域状态；READ 范围与 getImage 一致。 +3. 先核可见性/READ，再核 original 的 Admin，最后核 ready 和所选 variant 完整证据。key 必须精确匹配 `tenant/{tenant}/lab/{lab}/{variant}/{image_id}/{sha}.{ext}`；A 固定 PNG，O 对应 JPEG/PNG/WebP。SHA 小写 64 位，VersionId 非空/非 null、长度≤200、可打印 ASCII；引用不能来自客户端。 +4. 持有上述锁时 SDK 使用固定显式凭据作本地 SigV4，GET 参数为 Bucket、规范 Key、准确 VersionId，ExpiresIn=60。此处没有 HEAD/GET/PUT、网络调用、provider credential chain 或签名后改域名。签名 URL 再核实际 HTTPS origin/path、唯一 query、算法、credential scope、日期、60 秒、host signed header、版本、signature 格式；异常返回脱敏依赖错误。 +5. 应用复核有效期限；original 同事务 append image.download_original 审计：actor/tenant/image_id、variant、固定访问原因、request_id。无原图字节、URL、签名、token、对象 key/version 或 SHA 进入审计。analysis 不记录原图敏感读取审计。 +6. 成功提交后才将完整 grant 交给 HTTP，Cache-Control=no-store、Referrer-Policy=no-referrer。审计/提交失败则无 grant 到达调用者。图片、上传、任务、attempt、事件和幂等记录不变化；session idle touch 属于身份服务既有行为。 + +本地签名不保证对象存在或 IAM/反代可用。已登记版本被移除时 S3 GET 可能失败；API 不 HEAD，也不把图像改 rejected 或换成 latest。旧版本被新版本覆盖仍读取已登记准确版本，不能由对象 store 最新状态替代 DB 引用。 + +## 4. 签名能力与撤权 + +签名是 bearer 能力，不是实时 session 鉴权。撤权/禁用/删除提交后不再签发，既有 URL 最长仍可能在 60 秒内使用；不得声称撤 session 即撤所有已签 URL。签发时用户/图片锁可使并发撤权或删除等待短事务，但随后不能绕过新状态再次签发。 + +只把 URL 临时交给浏览器，原样使用并在页面设置 no-referrer；不写普通日志/监控 query/分析 SDK，不长期缓存。API 的 no-referrer header 不能替代前端页面和存储代理的部署策略。访问已完成/取消巡检的 ready 图像仍按 READ，与只读历史一致，不需要采集权限；原图仍额外 Admin。 + +本批不签发 staging/crop/reports GET，也不授予 list/delete 或新 IAM 通配范围；没有实现对象删除/紧急凭据撤销操作。 + +## 5. 本地验证 + +业务测试见 [test_image_download.py](../../tests/business/test_image_download.py)，真实 MySQL/HTTP/Redis 测试见 [test_image_download_mysql.py](../../tests/persistence/test_image_download_mysql.py)。新增 40 项业务用例、26 项真实 MySQL 用例。固定源图片、O/A 版本与结果为合成数据；本地 presign 使用真实 SDK/密码学，但不连接真实 MinIO。 + +| 验证 | 2026-10-02 实际结果 | +|---|---| +| 下载业务定向 | 40 passed；含独立 HMAC 复算 SigV4、方法/版本篡改核签反例 | +| 下载真实 MySQL 定向 | 26 passed，173.48s，无跳过；MySQL 8.0.33、43 表/115 FK、head=0001_initial | +| 业务/领域/安全/协议/持久化单元 | 1469 passed = 1444 主套件 + 4 协议 + 21 持久化单元，38.97s；1 项既有 Starlette/AnyIO 弃用警告 | +| 完整持久化 | 383 passed = 362 项真实 MySQL + 21 项单元，1212.38s，无跳过；1 项既有弃用警告；21 项单元与主套件重叠 | +| 最终响应与身份边界定向 | 49 passed，9.09s；包含 no-store/no-referrer 响应与既有 /ready 范围兼容 | +| 静态/格式/依赖 | Ruff check PASS；132 files format PASS；pip check PASS | +| 契约/设计/文档 | build_specs --check 16 制品未漂移；148 synthetic readiness；validate_specs PASS、277 文档链接 | + +主要风险覆盖：schema/默认 variant/strict query；四类实验室读者 analysis 可读而 original 拒绝；tenant/lab 404；非 ready/坏 key/hash/version 不调用 SDK；真实 URL 编码特殊 VersionId；签名绑定 GET/Host/path/version/60 秒并拒绝异常 URL；preflight 后撤会话/改角色/删图拒绝；签名期间另连接 FOR UPDATE NOWAIT 证明 user/image 锁有效；审计失败无返回 URL、无审计残留；每次原图读取独立审计；Redis 失败/429 不签发；限流在事务外。 + +### 可复现命令 + +~~~powershell +.\.venv-business\Scripts\python.exe -m pytest tests/business tests/domain tests/security tests/protocol tests/persistence/test_unit.py -q +$env:LABSAFE_TEST_REDIS_SERVER = '<专用 Redis 可执行文件路径>' +.\.venv-business\Scripts\python.exe tools/database/run_mysql_tests.py --mysqld '<专用 MySQL 8.x mysqld 路径>' +.\.venv-business\Scripts\python.exe -m ruff check apps packages tests tools/database +.\.venv-business\Scripts\python.exe -m ruff format --check apps packages tests tools/database +.\.venv-business\Scripts\python.exe -m pip check +python -B tools/design/build_specs.py --check +python -B tools/design/test_readiness_design.py +python -B tools/design/validate_specs.py +~~~ + +定向可临时 PYTEST_ADDOPTS='-x -k image_download'；完整回归移除过滤变量。MySQL/Redis 启动器只创建/清理自己的临时实例。完整持久化套件含 21 项与主套件重叠的单元，不累计成互不重叠的测试数量。AI/前端本批未修改；远程 CI 与独立审批未执行。 + +本地代码与验证批次已关闭；真实存储部署验收和完整系统阶段保持未完成。前序批次的验收数字保留为历史证据，不替换成 F4 的累计结果。 + +## 6. 回退与未关闭项 + +恢复 API_UPLOADS_ENABLED=0 并重启 API 可停止签发，同时关闭上传入口；不删除任何 DB/对象/审计。已签 URL 自然到期，不承诺即时失效；紧急事件按安全处置流程处理存储凭据。后台已登记任务不受签名入口停止影响。 + +真实部署 UP-03/SEC-03 仍需完成：versioned MinIO 同源 HTTPS 实际 GET 原/分析准确版本并比对 SHA；重复 PUT 不换已登记版本；篡改 method/Host/path/version/签名、过期 URL 的实际拒绝；跨 tenant/lab/staging/crop 的 IAM 负例；代理保持签名参数、访问日志不记录完整 query、页面 no-referrer、缓存策略与凭据撤销实测。SDK/HMAC/MySQL 成绩均不能代替以上验收。完整 I-02/I-03、crop/报告下载、业务提交与独立 AI 执行、生产发布仍待完成。 diff --git a/docs/08-delivery/22-i02g1-submit-inference-enqueue.md b/docs/08-delivery/22-i02g1-submit-inference-enqueue.md new file mode 100644 index 0000000..7af3cc2 --- /dev/null +++ b/docs/08-delivery/22-i02g1-submit-inference-enqueue.md @@ -0,0 +1,17 @@ +# I-02G1 `submitInspectionItem` 与推理任务入队验收 + +状态（2026-10-03）：本批在既有领域 `submit_item` guard 上接通真实 HTTP 写命令。请求只接受 `expected_version` 与有序 `images`；租户、实验室、父巡检、图片 owner/ready 状态、activation 和历史 findings 均由服务端在同一事务加载。 + +提交事务按 tenant/session → item → 当前 findings/images → activation 的顺序读取并锁定。成功时旧 run/evaluation/findings 被标记 superseded，item 的 `submission_revision` 与 version 增加，新 `inference_runs` 固定 published model/dictionary/rule、pipeline、device、UTC reference date 和 input hash，`run_images` 保存图像 SHA 与顺序；随后原子创建 `inference_pipeline` task、task-message-v1 `TaskDispatch` outbox、审计和幂等响应。客户端不得提交模型、规则、设备、日期、对象 key 或租户字段。 + +激活读取同时要求模型的 `dictionary_version_id` 与激活记录一致，并且模型、词典均为 `published`；不接受跨词典拼接的配置入队。 + +本批没有 AI 推理结果，也没有把任务标记 succeeded。统一 TaskDispatch 校验新增已有契约定义的 `inference_pipeline` 分支；发布器可从 outbox 投递该消息，但 Worker 的 inference lease、独立进程、超时、fencing、结果提交和重放仍属于后续 I-03/AI 批次。 + +验证入口： + +- `tests/persistence/test_inference_submit_mysql.py`：真实 MySQL activation、ready image、run/task/outbox 和 item pointer 验收； +- `tests/business`：路由、严格请求和既有调度协议回归； +- `tools/design/build_specs.py --check` 与 `tools/design/validate_specs.py`：契约/链接/就绪检查。 + +关闭条件：本批专门 MySQL 测试已通过；合并前仍需增加并发同 key/不同 key、stale version、跨租户/实验室、旧 run fencing、任一写步骤失败回滚用例。I-02G1 不关闭 JOB-04、独立 AI 进程或生产发布门禁。 diff --git a/docs/08-delivery/23-i03a2-inference-execution.md b/docs/08-delivery/23-i03a2-inference-execution.md new file mode 100644 index 0000000..555aa89 --- /dev/null +++ b/docs/08-delivery/23-i03a2-inference-execution.md @@ -0,0 +1,24 @@ +# 08.23 I-03A2：推理任务专属执行 + +状态（2026-10-04）：在既有 I-03A1/A2 调度基础上接入 `inference_pipeline` 的领域租约、attempt、心跳、围栏结果提交和过期回收。worker 通过独立 AI HTTP 协议调用 `/internal/inference/v1/quality` 与 `/runs`；默认关闭，仅 `APP_ENV=dev/test` 可用。真实模型、GPU、生产路由和规则评估仍未完成。 + +## 1. 本批交付 + +- `packages/domain/inference_execution.py` 定义固定输入、attempt/fencing lease、request hash、结果 Schema/租户/版本/输入 hash 校验及可重试错误白名单。 +- `packages/persistence/inference_execution.py` 按 item → current run → task 锁序领取 `inference_pipeline`，推进 `quality_checking/processing`，围栏提交 `needs_retake/needs_review`，技术失败进入 `retry_wait/dead_letter`，并支持过期租约回收。 +- `packages/application/inference_execution.py` 提供独立心跳和可注入 AI client；同一不可变请求先做 quality，再做 runs 复验，quality 结果不一致按 `SCHEMA_MISMATCH` 收敛。 +- `apps/worker/inference_pipeline.py` 与 worker 开关 `WORKER_INFERENCE_ENABLED=1` 接入 `q.general`；sweeper 同时回收 image/inference 执行租约。 + +AI 只返回事实，不直接写业务库。提交前必须同时匹配当前 run、submission revision、attempt id、fencing token、model/dictionary 版本及 SHA、pipeline 和有序图片 hash；旧 lease 或旧 run 结果被拒绝。`facts_ready` 会在同一围栏事务写入首个 `fact_revisions` 并把 run 推进到 `processing/facts`；规则评估任务和 `needs_review` 人工闭环仍属后续批次。 + +## 2. 验证 + +| 命令 | 结果 | +|---|---| +| `.venv-business\\Scripts\\python.exe -m pytest tests/business/test_inference_execution.py tests/business/test_dispatch.py tests/business/test_job_execution.py -q` | 56 passed | +| `.venv-business\\Scripts\\python.exe -m pytest tests/business tests/domain tests/protocol -q` | 1442 passed，1 个既有 Starlette/AnyIO 弃用警告 | +| `PYTEST_ADDOPTS='-k inference_execution' .venv-business\\Scripts\\python.exe tools/database/run_mysql_tests.py --mysqld ` | 1 passed；MySQL 8.0.33，43 表/115 外键，head=0001_initial | +| `.venv-business\\Scripts\\python.exe -m ruff check ...` | PASS | +| `.venv-business\\Scripts\\python.exe -m compileall ...` | PASS | + +本批尚未宣称真实 AI、MinIO/IAM/HTTPS、CUDA、facts/rules/report 完整闭环或 production 可用；worker consumer 仍需显式启用。 diff --git a/docs/08-delivery/24-i03a2-rule-evaluation.md b/docs/08-delivery/24-i03a2-rule-evaluation.md new file mode 100644 index 0000000..2f533b1 --- /dev/null +++ b/docs/08-delivery/24-i03a2-rule-evaluation.md @@ -0,0 +1,25 @@ +# 08.24 I-03A2：规则评估任务执行 + +状态（2026-10-05）：接通 `facts_ready` 后的 `rule_evaluation` TaskDispatch、专属 lease/attempt/heartbeat/fencing、rules-dnf-v1 三值求值和原子 evaluation/findings 提交，并补齐固定快照语义与事实上下文投影。规则 bundle 只从当前 run 固定的已发布成员装载；本批使用合成 DSL fixture 验证工程逻辑，不代表真实化学安全规则已批准。 + +## 1. 本批交付 + +- `packages/rules/evaluator.py` 校验 RuleBundle DSL，按作用域/有效时间选择规则，按 priority/rule_id 稳定排序，支持 `eq`、`in`、`before_reference_date` 和 true/false/unknown 三值传播；严格检查原子类型、case 去重、解释占位符及配对守卫。 +- `packages/persistence/rule_execution.py` 按 item→fact/evaluation→task 锁序领取规则任务，从 bundle 成员固定装载 published rule_versions、evaluator/checksum 快照，组装 expiry/storage/location/adjacency/subject 上下文，生成 needs_review findings，写入 evaluation result/hash/evidence，并收敛 run/item。 +- `packages/application/rule_execution.py` 在事务外执行纯规则计算，提交时重新锁定并进行租约、generation、token 和当前指针围栏校验。 +- `facts_ready` 结果现在同事务创建初始 `fact_revisions`、排队 `rule_evaluations`、`rule_evaluation` task 和 TaskDispatch outbox。 +- durable dispatch 支持 `rule_evaluation` 消息；Worker 和 sweeper 增加显式 `WORKER_RULE_EVALUATION_ENABLED=1` 分支。 + +规则永久错误（DSL 无效、容量超限）不会自动重试；技术失败沿用专属 task retry/dead-letter。旧 generation、旧 token、旧 run、过期 lease 不写当前评估或 findings。零 finding 仍进入人工复核，不能自动完成。 + +## 2. 验证 + +| 命令 | 结果 | +|---|---| +| `.venv-business\\Scripts\\python.exe -m pytest tests/business/test_rule_evaluator.py tests/business/test_dispatch.py -q` | 37 passed | +| `PYTEST_ADDOPTS='-k inference_execution' .venv-business\\Scripts\\python.exe tools/database/run_mysql_tests.py --mysqld ` | 1 passed；MySQL 8.0.33，43 表/115 外键，含 facts→rules→findings 闭环 | +| `.venv-business\\Scripts\\python.exe -m ruff check ...` | PASS | +| `.venv-business\\Scripts\\python.exe -m pytest tests/business tests/domain tests/protocol -q` | 1451 passed；1 个既有 Starlette/AnyIO 弃用警告 | +| `tools/design/build_specs.py --check` / `test_readiness_design.py` / `validate_specs.py` | 全部 PASS；148 synthetic readiness checks | + +真实安全规则审批、规则管理 API、report/export、真实生产发布和完整 remediation 闭环仍未完成。 diff --git a/docs/08-delivery/25-i02-rule-management.md b/docs/08-delivery/25-i02-rule-management.md new file mode 100644 index 0000000..7df720a --- /dev/null +++ b/docs/08-delivery/25-i02-rule-management.md @@ -0,0 +1,31 @@ +# 08.25 I-02：规则配置管理 + +状态（2026-10-05）：在规则执行快照基础上接通开发/测试环境规则配置 API。范围覆盖规则集创建与列表、规则版本导入、提交审批、批准、发布、退休、固定 bundle 创建与列表；所有写命令使用租户事务、幂等键、乐观版本、审计和发布状态守卫。 + +本批不批准任何真实化学安全规则，也不开放 production。规则 DSL 仍由 `rules-dnf-v1` evaluator 和 `contracts/rule-dsl-v1.json` 校验；bundle 只接受同租户 `published` rule_versions，按版本 ID 排序计算 checksum，成员创建后不可变。审批要求 `rule_expert`，且提交者不能批准自己的版本；旧 run 继续使用已固定的 bundle。 + +## 1. API + +| 方法 | 路径 | 事务语义 | +|---|---|---| +| POST/GET | `/rule-sets` | safety_admin 创建；租户可见列表 | +| GET/POST | `/rule-versions` | published 版本列表；admin 导入 draft | +| GET | `/rule-versions/{id}` | 当前租户可见投影，草稿仅 admin/rule_expert | +| POST | `/rule-versions/{id}/submit-approval` | draft → submitted | +| POST | `/rule-versions/{id}/approve` | submitted → approved，审批人需 rule_expert 且不同于提交人 | +| POST | `/rule-versions/{id}/publish` | approved → published | +| POST | `/rule-versions/{id}/retire` | published → retired，仅影响新 bundle | +| GET/POST | `/rule-bundles` | 固定 published 成员；重复 checksum 幂等复用 | + +`packages/persistence/rules.py` 负责行锁、状态转移、checksum、成员投影和审计；`packages/application/rules.py` 负责身份、租户锁、幂等响应和死锁重试；`apps/api/app/rules.py` 只做公共输入/路由适配。 + +## 2. 验证 + +| 命令 | 结果 | +|---|---| +| `.venv-business\\Scripts\\python.exe -m pytest tests/business -q` | 393 passed；1 个既有 Starlette/AnyIO 弃用警告 | +| `.venv-business\\Scripts\\python.exe -m ruff check ...` | PASS | +| `.venv-business\\Scripts\\python.exe -m ruff format --check ...` | PASS | +| `tools/design/build_specs.py --check` / `test_readiness_design.py` / `validate_specs.py` | PASS | + +隔离 MySQL 的规则管理专项仍需补充；当前环境未提供本批专用 `mysqld` 路径，因此未把跳过的集成测试记为通过。人工事实修订和 finding 决策已在 [I-02 事实修订与人工复核](26-i02-review.md) 接通；remediation、report/export 和非图像任务 replay 仍属后续批次。 diff --git a/docs/08-delivery/26-i02-review.md b/docs/08-delivery/26-i02-review.md new file mode 100644 index 0000000..6f342b0 --- /dev/null +++ b/docs/08-delivery/26-i02-review.md @@ -0,0 +1,25 @@ +# 08.26 I-02：事实修订与人工复核 + +状态（2026-10-05）：在规则评估和规则配置批次之后，接通当前事实读取、人工事实完整快照修订、finding 查询以及 confirm/reject/cannot-determine 决策。实现仍只允许 dev/test，合成规则和 fixture 不代表真实化学安全规则已批准。 + +## 1. 本批交付 + +- `GET /inspection-items/{id}/facts` 返回当前 run 的事实；未建立 current pointer 时返回 `404`，读取只使用服务端租户/实验室授权。 +- `POST /inspection-items/{id}/facts` 只接受 `needs_review` item 的完整 `entities/relations/dates` 快照。事务锁顺序为 item→run→当前 findings→evaluation，拒绝已确认/已派发/已关闭 finding；只允许当前 run 的 detection、image、crop 和固定 dictionary entity，所有事实来源必须是 `human`。 +- 事实修订追加 `fact_revisions`，将旧 evaluation/findings 标记 superseded，创建新的 queued `rule_evaluation`、`task_runs` 和 `TaskDispatch` outbox，并更新 item/run current pointers；旧 lease、旧 generation 和旧 evaluation 不能复活当前结果。 +- `GET /findings` 默认 `current_only=true`，支持实验室、状态和严重度过滤；finding projection 固定返回非空 `EvidenceRef[]`、task_id、superseded_at 和可信 `allowed_actions`。 +- `POST /findings/{id}/confirm`、`reject`、`cannot-determine` 使用 domain guard、expected_version、review_actions、audit 和幂等响应；confirm 需要 evidence，不能对 superseded 或已处理 finding 重复操作。 +- API 层使用严格 body/query allowlist；application 层负责身份、租户锁、幂等、死锁重试，持久层负责锁定完整写集和原子提交。 + +## 2. 验证 + +| 命令 | 结果 | +|---|---| +| `.venv-business\\Scripts\\python.exe -m pytest tests/business/test_review_boundary.py tests/business/test_rules_boundary.py -q` | 26 passed；1 个既有 Starlette/AnyIO 弃用警告 | +| `.venv-business\\Scripts\\ruff.exe check packages/persistence/review.py packages/application/review.py apps/api/app/review.py apps/api/app/main.py packages/persistence/foundations.py tests/business/test_review_boundary.py` | PASS | +| `APP_ENV=test .venv-business\\Scripts\\python.exe -m compileall -q apps/api/app packages/application packages/persistence` | PASS | +| API operation id smoke | 54 个 route，54 个唯一 operation id；新增 7 个复核 operation | +| `.venv-business\\Scripts\\python.exe -m pytest tests/business tests/domain tests/protocol -q` | 1477 passed；1 个既有 Starlette/AnyIO 弃用警告 | +| `tools/database/run_mysql_tests.py --mysqld <专用 mysqld>` | 386 passed；MySQL 8.0.33、43 表、115 外键、head=0001_initial；1 个既有弃用警告 | + +本批真实 MySQL 使用独立临时 data directory、随机回环端口和专用 schema，测试结束后由 runner 清理；未连接或修改既有数据库。remediation、report/export、完整 item complete、真实存储部署验收和生产发布仍未完成。新增领域事件已写入 outbox，但通用领域事件 publisher 尚未接入，因此不宣称已实际发布。 diff --git a/docs/08-delivery/README.md b/docs/08-delivery/README.md index 79c21e6..9e05f3d 100644 --- a/docs/08-delivery/README.md +++ b/docs/08-delivery/README.md @@ -16,5 +16,15 @@ 14. [I-02E巡检项查询与动作投影基础验收](14-i02e-item-query-api.md)。 15. [I-02F1上传授权与S3适配基础验收](15-i02f1-upload-grant-api.md)。 16. [I-02F2上传完成受理与图像查询验收](16-i02f2-upload-completion-api.md)。 +17. [I-03A1上传验证任务持久调度](17-i03a1-durable-dispatch.md)。 +18. [I-03A2上传验证任务执行基础](18-i03a2-image-execution.md)。 +19. [I-02F3图像验证处理与原子结果](19-i02f3-image-validation.md)。 +20. [I-03A3图像任务查询与管理员重放](20-i03a3-image-job-replay.md)。 +21. [I-02F4受控图片下载与原图审计](21-i02f4-image-download.md)。 +22. [I-02G1提交巡检项与推理任务入队](22-i02g1-submit-inference-enqueue.md)。 +23. [I-03A2推理任务专属执行](23-i03a2-inference-execution.md)。 +24. [I-03A2规则评估任务执行](24-i03a2-rule-evaluation.md)。 +25. [I-02规则配置管理](25-i02-rule-management.md)。 +26. [I-02事实修订与人工复核](26-i02-review.md)。 本页仅导航；现行设计状态见 [文档控制](../00-document-control.md)，不把候选设计视为已经批准。 diff --git a/docs/README.md b/docs/README.md index 8d49b13..85f5fc8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,11 +1,11 @@ # LabSafe 文档入口 -状态：设计版本 1.1.0；实现进度更新于 2026-10-01。设计结论见 [最新审计](08-delivery/03-design-audit.md)，不要把结构检查或局部实现通过当作全系统可上线。 +状态：设计版本 1.1.0；实现进度更新于 2026-10-03。设计结论见 [最新审计](08-delivery/03-design-audit.md)，不要把结构检查或局部实现通过当作全系统可上线。 ## 阅读顺序 -实现进度：I-01A/B/C 已合并；I-02A/B/C/D/E/F1/F2 通过本地相应验证，本次提交特性分支 PR，完整 I-02 仍未完成。审批状态、远程 CI 与本机测试分别记录。 -运行入口见 [开发指南](../DEVELOPMENT.md)，本轮证据见 [I-02F2 上传完成受理验收](08-delivery/16-i02f2-upload-completion-api.md)，完整阶段状态见下方实施计划。 +实现进度：I-01A/B/C、I-02A/B/C/D/E/F1/F2 已合并（后者为 PR #9）；当前在 I-03A1/A2 调度与围栏、I-02F3 general 图像处理、I-03A3 图像任务查询/管理员重放上接通 I-02F4 ready 图像受控下载，并已接通 I-02G1 `submitInspectionItem` 的 run/task/outbox 原子入队；代码本地完成、真实存储部署验收与推理 worker 仍待关闭，完整 I-02/I-03 仍未完成。审批状态、远程 CI 与本机测试分别记录。 +运行入口见 [开发指南](../DEVELOPMENT.md)，本轮证据见 [I-02G1 提交入队验收](08-delivery/22-i02g1-submit-inference-enqueue.md)，前序证据见 [I-02F4 图片下载验收](08-delivery/21-i02f4-image-download.md) 与 [I-03A3 图像任务重放](08-delivery/20-i03a3-image-job-replay.md)，完整阶段状态见下方实施计划。 1. [文档控制](00-document-control.md)：规范边界和变更要求。 2. [产品](01-requirements/01-product-requirements.md)与[范围](01-requirements/01-scope-and-acceptance.md)。 diff --git a/packages/application/dispatch.py b/packages/application/dispatch.py new file mode 100644 index 0000000..1edcf5d --- /dev/null +++ b/packages/application/dispatch.py @@ -0,0 +1,70 @@ +"""Publish outside transactions; uncertain delivery deliberately remains retryable.""" + +import logging +from contextlib import contextmanager +from uuid import uuid4 + +from packages.persistence import dispatch + +LOG = logging.getLogger(__name__) + + +@contextmanager +def transaction(engine): + # RC avoids scan-index gap locks; leave API RR connections unchanged. + with engine.connect() as connection: + connection = connection.execution_options(isolation_level="READ COMMITTED") + with connection.begin(): + yield connection + + +class DispatchPublisher: + def __init__(self, engine, transport): + self.engine = engine + self.transport = transport + + def publish_batch(self, limit=100): + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("Batch limit must be 1..100") + published = 0 + for _ in range(limit): + owner = str(uuid4()) + with transaction(self.engine) as connection: + row = dispatch.claim(connection, owner) + if row is None: + break + + def renew(): + with transaction(self.engine) as connection: + if not dispatch.owned_update(connection, row, owner, "renew"): + raise RuntimeError("Publication lease lost") + + try: + message = dispatch.validate_dispatch(dispatch.decoded(row["payload"])) + if ( + message["tenant_id"] != row["tenant_id"] + or message["task_id"] != row["aggregate_id"] + or row["aggregate_type"] != "task_run" + ): + raise ValueError("Outbox envelope mismatch") + # Transport must enforce a 20s wall deadline and heartbeat every 10s. + self.transport.publish(row["id"], message, renew) + with transaction(self.engine) as connection: + accepted = dispatch.owned_update(connection, row, owner, "published") + published += int(accepted) + if not accepted: + LOG.warning("dispatch_lease_lost event_id=%s", row["id"]) + except Exception: + # No exception body/payload/URL in logs; they may include secrets. + LOG.error("dispatch_publish_failed event_id=%s", row["id"]) + with transaction(self.engine) as connection: + dispatch.owned_update(connection, row, owner, "release") + return published + + +def sweep_dispatch(engine): + with transaction(engine) as connection: + recovered = dispatch.recover_publications(connection) + with transaction(engine) as connection: + resent = dispatch.redispatch(connection) + return recovered, resent diff --git a/packages/application/image_download.py b/packages/application/image_download.py new file mode 100644 index 0000000..fcbd100 --- /dev/null +++ b/packages/application/image_download.py @@ -0,0 +1,50 @@ +"""Local signing under current authorization locks; original grant and audit commit together.""" + +from packages.application.identity import retry_deadlocks +from packages.domain.image_download import DOWNLOAD_SECONDS, download_reference +from packages.domain.security import ServiceError +from packages.persistence.images import ImageRepository +from packages.persistence.security import append_audit, utc_now +from packages.persistence.users import timestamp + + +class ImageDownloadApplication: + def __init__(self, identity, storage): + self.identity, self.storage = identity, storage + self.images = ImageRepository() + + def download(self, token, image_id, variant, request_id): + if variant not in {"analysis", "original"}: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid download variant") + actor = self.identity._preflight(token) + # Both variants issue bearer capabilities; Redis failure must not use read fallback. + self.identity.limits.download_grant(actor) + return self._download_once(token, image_id, variant, request_id) + + @retry_deadlocks + def _download_once(self, token, image_id, variant, request_id): + with self.identity.transaction() as connection: + actor = self.identity.sessions.authenticate(connection, token).principal + reference = download_reference( + actor, self.images.download_row(connection, actor, image_id), variant + ) + # Explicit fixed credentials: local cryptography only. Never HEAD/GET here. + grant = self.storage.presign_image_get(reference) + remaining = (grant.expires_at - utc_now()).total_seconds() + if not 0 < remaining <= DOWNLOAD_SECONDS: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Invalid download lifetime") + if variant == "original": + append_audit( + connection, + principal=actor, + action="image.download_original", + resource_type="image", + resource_id=image_id, + changes={"variant": "original"}, + reason="Authorized original image download", + request_id=request_id, + ) + return { + "data": {"url": grant.url, "expires_at": timestamp(grant.expires_at)}, + "request_id": request_id, + } diff --git a/packages/application/image_validation.py b/packages/application/image_validation.py new file mode 100644 index 0000000..5a805d7 --- /dev/null +++ b/packages/application/image_validation.py @@ -0,0 +1,77 @@ +"""Real image preparation. Object I/O and decoding never receive a DB connection.""" + +import hashlib +import warnings +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + +from packages.domain.image_validation import ( + CONTENT_ERRORS, + RejectedImage, + ValidatedImage, + image_keys, +) +from packages.domain.security import ServiceError +from packages.storage.s3 import ObjectVersion + +FORMATS = {"JPEG": "image/jpeg", "PNG": "image/png", "WEBP": "image/webp"} + + +def dimensions(width, height): + if not (1 <= width <= 10000 and 1 <= height <= 10000 and width * height <= 40_000_000): + raise ServiceError("IMAGE_TOO_LARGE", 422, "Image dimensions exceed limits") + + +def normalize(payload, mime): + """Fully decode one static image; transpose once; create metadata-free RGB PNG.""" + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(BytesIO(payload)) as probe: + dimensions(*probe.size) + if FORMATS.get(probe.format) != mime: + raise ServiceError("UNSUPPORTED_MEDIA_TYPE", 422, "Image format mismatch") + if getattr(probe, "n_frames", 1) != 1: + raise ServiceError("IMAGE_INVALID", 422, "Animated images are unsupported") + probe.verify() + with Image.open(BytesIO(payload)) as source: + source.load() # Keep Pillow's default rejection of truncated input. + with ImageOps.exif_transpose(source) as oriented: + with oriented.convert("RGB") as rgb: + dimensions(*rgb.size) + # frombytes discards every info/EXIF/ICC/text field, including GPS. + with Image.frombytes("RGB", rgb.size, rgb.tobytes()) as clean: + output = BytesIO() + clean.save(output, format="PNG", compress_level=6, optimize=False) + return output.getvalue(), clean.width, clean.height + except ServiceError: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise ServiceError("IMAGE_TOO_LARGE", 422, "Image dimensions exceed limits") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError): + raise ServiceError("IMAGE_INVALID", 422, "Image cannot be decoded") from None + + +def prepare_image(storage, tenant, input): + pinned = ObjectVersion(input.key, input.object_version, input.size_bytes, input.mime_type) + try: + original = storage.read_staging(tenant, input.upload_id, pinned, input.expected_sha256) + analysis, width, height = normalize(original, input.mime_type) + analysis_sha = hashlib.sha256(analysis).hexdigest() + original_key, analysis_key = image_keys(tenant, input, analysis_sha) + original_version = storage.preserve_original(pinned, original_key, input.expected_sha256) + analysis_version = storage.put_analysis(analysis_key, analysis) + return ValidatedImage( + original_key, + original_version, + analysis_key, + analysis_version, + analysis_sha, + width, + height, + ) + except ServiceError as error: + if error.code in CONTENT_ERRORS: + return RejectedImage(error.code) + raise diff --git a/packages/application/inference_execution.py b/packages/application/inference_execution.py new file mode 100644 index 0000000..f6900eb --- /dev/null +++ b/packages/application/inference_execution.py @@ -0,0 +1,208 @@ +"""Worker-side inference orchestration with bounded, injectable AI RPC.""" + +import json +import os +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone +from threading import Event, Thread +from uuid import uuid4 + +from packages.application.dispatch import transaction +from packages.domain.inference_execution import ( + InferenceInvalidResult, + InferenceLeaseLost, + request_payload, + validate_result, +) +from packages.inference_protocol.contract import validate +from packages.persistence import inference_execution +from packages.shared.environment import development_environment + + +class AIRequestError(RuntimeError): + def __init__(self, code): + self.code = code + super().__init__(code) + + +class InferenceHTTPClient: + """Small stdlib client; tests may inject an object with quality/runs methods.""" + + def __init__(self, endpoint=None, token=None, timeout=200): + self.endpoint = ( + endpoint or os.environ.get("AI_INFERENCE_URL", "http://127.0.0.1:8001") + ).rstrip("/") + self.token = token if token is not None else _read_token() + self.timeout = timeout + + def _post(self, path, payload): + request = urllib.request.Request( + f"{self.endpoint}/internal/inference/v1/{path}", + data=json.dumps(payload, separators=(",", ":")).encode(), + headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + value = json.loads(response.read()) + except urllib.error.HTTPError as error: + try: + body = json.loads(error.read()) + code = body.get("error", {}).get("code", "DEPENDENCY_UNAVAILABLE") + except Exception: + code = "DEPENDENCY_UNAVAILABLE" + raise AIRequestError(code) from None + except (urllib.error.URLError, TimeoutError, OSError): + raise AIRequestError("DEPENDENCY_UNAVAILABLE") from None + try: + validate("InferenceResult", value) + except ValueError: + raise AIRequestError("SCHEMA_MISMATCH") from None + return value + + def quality(self, payload): + return self._post("quality", payload) + + def runs(self, payload): + return self._post("runs", payload) + + +def _read_token(): + path = os.environ.get("AI_TOKEN_FILE") + if not path: + raise RuntimeError("AI_TOKEN_FILE is required for inference worker") + try: + value = open(path, encoding="utf-8").read().strip() + except (OSError, UnicodeError) as exc: + raise RuntimeError("Cannot read AI_TOKEN_FILE") from exc + if not value or not value.isascii() or any(char.isspace() for char in value): + raise RuntimeError("Invalid AI token") + return value + + +class InferenceHeartbeat: + interval = 10 + + def __init__(self, execution, lease, cancel): + self.execution, self.lease, self.cancel = execution, lease, cancel + self.stop = Event() + self.lost = Event() + self.thread = Thread(target=self._loop, name="inference-lease-heartbeat", daemon=True) + + def _loop(self): + while not self.stop.wait(self.interval): + try: + renewed = self.execution.heartbeat(self.lease) + except Exception: + renewed = False + if renewed: + continue + self.lost.set() + try: + self.cancel() + except Exception: + pass + return + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, exc_type, exc, traceback): + self.stop.set() + self.thread.join(timeout=2) + if self.thread.is_alive() or self.lost.is_set(): + raise InferenceLeaseLost() from None + return False + + +class InferenceExecution: + def __init__(self, engine, client=None): + development_environment() + self.engine = engine + # Sweepers only recover leases and must not require an AI token file. + self.client = client + + def claim(self, message): + with transaction(self.engine) as connection: + return inference_execution.claim(connection, message, str(uuid4())) + + def heartbeat(self, lease): + with transaction(self.engine) as connection: + return inference_execution.heartbeat(connection, lease) + + def fail(self, lease, code): + with transaction(self.engine) as connection: + return inference_execution.fail(connection, lease, code) + + def commit(self, lease, result): + with transaction(self.engine) as connection: + return inference_execution.commit_result(connection, lease, result) + + def recover_expired(self, limit=100): + with transaction(self.engine) as connection: + candidates = inference_execution.expired_candidates(connection, limit) + recovered = 0 + for candidate in candidates: + with transaction(self.engine) as connection: + recovered += int(inference_execution.recover(connection, candidate)) + return recovered + + def execute(self, message): + lease = self.claim(message) + if lease is None: + return False + cancelled = Event() + try: + with InferenceHeartbeat(self, lease, cancelled.set): + client = self.client or InferenceHTTPClient() + quality_payload = request_payload( + lease, datetime.now(timezone.utc) + timedelta(seconds=15) + ) + quality = client.quality(quality_payload) + validate_result(quality, lease) + if any( + quality[name] + for name in ("detections", "crops", "ocr_fields", "entities", "relations") + ): + raise AIRequestError("SCHEMA_MISMATCH") + if quality["outcome"] == "needs_retake": + if not any(item["status"] == "needs_retake" for item in quality["quality"]): + raise AIRequestError("SCHEMA_MISMATCH") + result = quality + else: + if quality["outcome"] != "facts_ready": + raise AIRequestError("SCHEMA_MISMATCH") + if any(item["status"] != "pass" for item in quality["quality"]): + raise AIRequestError("SCHEMA_MISMATCH") + # The runs call uses the same immutable request content and + # attempt fencing; only the absolute deadline changes. + runs_payload = request_payload( + lease, datetime.now(timezone.utc) + timedelta(seconds=200) + ) + result = client.runs(runs_payload) + validate_result(result, lease) + if result["quality"] != quality["quality"]: + raise AIRequestError("SCHEMA_MISMATCH") + if result["outcome"] == "needs_retake" and not any( + item["status"] == "needs_retake" for item in result["quality"] + ): + raise AIRequestError("SCHEMA_MISMATCH") + if result["outcome"] != "needs_retake" and any( + item["status"] == "needs_retake" for item in result["quality"] + ): + raise AIRequestError("SCHEMA_MISMATCH") + self.commit(lease, result) + except InferenceLeaseLost: + raise + except AIRequestError as error: + self.fail(lease, error.code) + raise + except InferenceInvalidResult: + self.fail(lease, "SCHEMA_MISMATCH") + raise + except Exception: + self.fail(lease, "INTERNAL_ERROR") + raise + return True diff --git a/packages/application/inference_runs.py b/packages/application/inference_runs.py new file mode 100644 index 0000000..073854f --- /dev/null +++ b/packages/application/inference_runs.py @@ -0,0 +1,79 @@ +"""Application boundary for submitInspectionItem.""" + +from datetime import timedelta +from uuid import UUID, uuid4 + +from sqlalchemy import text + +from packages.application.identity import retry_deadlocks +from packages.domain.security import Permission, ServiceError, authorize +from packages.domain.workflow import ImageSelection +from packages.persistence.inference_runs import submit +from packages.persistence.security import begin_idempotency, complete_idempotency, utc_now + + +class InferenceRunApplication: + def __init__(self, identity): + self.identity = identity + + def submit(self, token, csrf, key, item_id, body, request_id): + actor = self.identity._preflight(token, csrf=csrf) + self.identity.limits.user(actor, write=True) + return self._submit_once(token, csrf, key, item_id, body, request_id) + + @retry_deadlocks + def _submit_once(self, token, csrf, key, item_id, body, request_id): + try: + item_id = str(UUID(item_id)) + if not isinstance(body, dict) or set(body) != {"expected_version", "images"}: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid item submit body") + selections = tuple( + ImageSelection( + image_id=str(UUID(value["image_id"])), + role=value["role"], + parent_image_id=( + str(UUID(value["parent_image_id"])) if value["parent_image_id"] else None + ), + ) + for value in body["images"] + ) + except (KeyError, TypeError, ValueError): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid item submit body") from None + with self.identity.transaction() as connection: + tenant, actor_id = self.identity.sessions.locate(connection, token) + owner = str(uuid4()) + claim = begin_idempotency( + connection, + tenant_id=tenant, + actor_id=actor_id, + method="POST", + path=f"/api/v1/inspection-items/{item_id}/submit", + key=key, + request_hash=self.identity.sessions.request_digest(body), + expires_at=utc_now() + timedelta(hours=24), + lease_owner=owner, + ) + session = self.identity.sessions.authenticate(connection, token) + self.identity.sessions.require_csrf(session, csrf) + actor = session.principal + if claim.state == "completed": + # Re-check visibility only; do not rerun state/version guards. + cached = connection.execute( + text( + "SELECT laboratory_id FROM inspection_items " + "WHERE tenant_id=:tenant AND id=:id FOR SHARE" + ), + {"tenant": actor.tenant_id, "id": item_id}, + ).first() + if cached is None: + raise ServiceError("NOT_FOUND", 404, "Not found") + authorize(actor, Permission.CAPTURE, cached[0]) + return claim.response_status, claim.response + data = submit( + connection, actor, item_id, body["expected_version"], selections, request_id + ) + response = {"data": data, "request_id": request_id} + complete_idempotency( + connection, claim, status=202, response=response, lease_owner=owner + ) + return 202, response diff --git a/packages/application/job_execution.py b/packages/application/job_execution.py new file mode 100644 index 0000000..ef25d2f --- /dev/null +++ b/packages/application/job_execution.py @@ -0,0 +1,118 @@ +"""Private execution service and independent lease heartbeat for future handlers.""" + +import logging +from threading import Event, Thread +from uuid import uuid4 + +from packages.application.dispatch import transaction +from packages.domain.job_execution import TECHNICAL_ERRORS, LeaseLost +from packages.domain.security import ServiceError +from packages.persistence import job_execution +from packages.shared.environment import development_environment + +LOG = logging.getLogger(__name__) + + +class ImageExecution: + def __init__(self, engine): + development_environment() + self.engine = engine + + def claim(self, message): + with transaction(self.engine) as connection: + return job_execution.claim(connection, message, str(uuid4())) + + def heartbeat(self, lease): + with transaction(self.engine) as connection: + return job_execution.heartbeat(connection, lease) + + def fail(self, lease, code): + with transaction(self.engine) as connection: + return job_execution.fail(connection, lease, code) + + def commit(self, lease, write_result): + with transaction(self.engine) as connection: + job_execution.commit_result(connection, lease, write_result) + + def execute(self, message, prepare, write_result): + """Private handler integration; external prepare never receives a DB connection. + + prepare(input, cancelled) returns an outcome (including content rejection) + or raises a technical ServiceError. write_result(conn, input, outcome) + owns all domain/audit/outbox writes. F3 supplies the private server handler. + """ + lease = self.claim(message) + if lease is None: + return False + cancelled = Event() + try: + with LeaseHeartbeat(self, lease, cancelled.set): + outcome = prepare(lease.input, cancelled) + self.commit(lease, lambda connection, input: write_result(connection, input, outcome)) + except LeaseLost: + raise + except Exception as error: + code = error.code if isinstance(error, ServiceError) else "INTERNAL_ERROR" + if code not in TECHNICAL_ERRORS: + code = "INTERNAL_ERROR" + self.fail(lease, code) + raise + return True + + def recover_expired(self, limit=100): + with transaction(self.engine) as connection: + candidates = job_execution.expired_candidates(connection, limit) + recovered = 0 + for candidate in candidates: + # Each task owns a short domain-first transaction. A second sweeper + # rechecks token/expiry and becomes a no-op after the first commits. + with transaction(self.engine) as connection: + recovered += int(job_execution.recover(connection, candidate)) + return recovered + + +class LeaseHeartbeat: + """A connection per renewal; a failure latches until this attempt is discarded. + + Use around external I/O, then leave the context and call commit(). Callback + cancellation must be nonblocking. No model calculation belongs on this thread. + """ + + interval = 10 + + def __init__(self, execution, lease, cancel): + self.execution, self.lease, self.cancel = execution, lease, cancel + self.stop = Event() + self.lost = Event() + self.thread = Thread(target=self._loop, name="task-lease-heartbeat", daemon=True) + + def _loop(self): + while not self.stop.wait(self.interval): + try: + if self.execution.heartbeat(self.lease): + continue + except Exception: + pass + self.lost.set() + LOG.warning("task_lease_lost task_id=%s", self.lease.task_id) + try: + self.cancel() + except Exception: + LOG.warning("task_cancel_failed task_id=%s", self.lease.task_id) + return + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, exc_type, exc, traceback): + self.stop.set() + self.thread.join(timeout=2) + if self.thread.is_alive(): + # A stuck database operation must never allow the caller to commit. + self.lost.set() + if self.lost.is_set(): + # Cancellation often makes prepare raise. That exception must not + # turn lost ownership into a new technical-failure write. + raise LeaseLost() from None + return False diff --git a/packages/application/jobs.py b/packages/application/jobs.py new file mode 100644 index 0000000..93938c6 --- /dev/null +++ b/packages/application/jobs.py @@ -0,0 +1,103 @@ +"""Manual image-job replay: authenticate, observe fixed input outside locks, revalidate, commit.""" + +from datetime import timedelta +from uuid import uuid4 + +from packages.application.identity import retry_deadlocks +from packages.domain.job_replay import replay_body, require_replay +from packages.domain.security import Permission, ServiceError, authorize +from packages.persistence.jobs import JobRepository +from packages.persistence.security import begin_idempotency, complete_idempotency, utc_now +from packages.persistence.transaction import lock_tenant +from packages.storage.s3 import ObjectVersion + + +class _NeedsPinnedInspection(Exception): + def __init__(self, tenant, input): + self.tenant, self.input = tenant, input + + +class JobApplication: + def __init__(self, identity, storage=None): + self.identity, self.storage = identity, storage + self.jobs = JobRepository() + + def read( + self, operation, token, request_id, *, job_id=None, page=1, page_size=20, laboratory_id=None + ): + actor = self.identity._preflight(token, admin=operation == "dead_letters") + self.identity.limits.user(actor, write=False) + return self._read_once(operation, token, request_id, job_id, page, page_size, laboratory_id) + + @retry_deadlocks + def _read_once(self, operation, token, request_id, job_id, page, size, lab): + with self.identity.transaction() as connection: + actor = self.identity.sessions.authenticate(connection, token).principal + if operation == "get": + data = self.jobs.get(connection, actor, job_id) + elif operation == "dead_letters": + data = self.jobs.dead_letters(connection, actor, page, size, lab) + else: + raise ValueError("Unknown job query") + return {"data": data, "request_id": request_id} + + def replay(self, token, csrf, key, job_id, body, request_id): + replay_body(body) + actor = self.identity._preflight(token, csrf=csrf, admin=True) + self.identity.limits.user(actor, write=True) + try: + return self._replay_once(token, csrf, key, job_id, body, request_id) + except _NeedsPinnedInspection as needed: + tenant, input = needed.tenant, needed.input + if self.storage is None: + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "Pinned storage check not configured") + reference = ObjectVersion( + input.key, input.object_version, input.size_bytes, input.mime_type + ) + try: + self.storage.inspect_pinned_staging(tenant, input.upload_id, reference) + except ServiceError as error: + if error.code == "OBJECT_NOT_FOUND": + raise ServiceError("STATE_CONFLICT", 409, "Pinned input no longer exists") from None + raise + return self._replay_once(token, csrf, key, job_id, body, request_id, observed=input) + + @retry_deadlocks + def _replay_once(self, token, csrf, key, job_id, body, request_id, *, observed=None): + with self.identity.transaction() as connection: + tenant, user = self.identity.sessions.locate(connection, token) + lock_tenant(connection, tenant) + lease = str(uuid4()) + claim = begin_idempotency( + connection, + tenant_id=tenant, + actor_id=user, + method="POST", + path=f"/api/v1/dead-letters/{job_id}/replay", + key=key, + request_hash=self.identity.sessions.request_digest(body), + expires_at=utc_now() + timedelta(hours=24), + lease_owner=lease, + ) + session = self.identity.sessions.authenticate(connection, token) + self.identity.sessions.require_csrf(session, csrf) + actor = session.principal + authorize(actor, Permission.ADMIN) + if claim.state == "completed": + # Recheck current visibility/admin, not obsolete state/version or object lifetime. + self.jobs.get(connection, actor, job_id) + return claim.response_status, claim.response + task, current = self.jobs.replay_source(connection, actor, job_id) + require_replay(actor, task, current, body["expected_version"]) + if observed is None: + raise _NeedsPinnedInspection( + tenant, current + ) # Rolls back provisional claim and locks. + if current != observed: + raise ServiceError("STATE_CONFLICT", 409, "Pinned input changed during inspection") + data = self.jobs.replay(connection, actor, task, body["reason"], request_id) + response = {"data": data, "request_id": request_id} + complete_idempotency( + connection, claim, status=202, response=response, lease_owner=lease + ) + return 202, response diff --git a/packages/application/rate_limit.py b/packages/application/rate_limit.py index 38d0633..c2a4bd1 100644 --- a/packages/application/rate_limit.py +++ b/packages/application/rate_limit.py @@ -91,3 +91,7 @@ def user(self, principal, *, write): def upload_grant(self, principal): self._charge("upload", f"{principal.tenant_id}:{principal.user_id}", 10, 60) + + def download_grant(self, principal): + # Read quota, but bearer-capability issuance must fail closed on Redis loss. + self._charge("user", f"{principal.tenant_id}:{principal.user_id}", 120, 60) diff --git a/packages/application/review.py b/packages/application/review.py new file mode 100644 index 0000000..3c111bf --- /dev/null +++ b/packages/application/review.py @@ -0,0 +1,138 @@ +"""Application transactions for fact review and finding decisions.""" + +from datetime import timedelta +from uuid import UUID, uuid4 + +from packages.application.identity import retry_deadlocks +from packages.domain.security import ServiceError +from packages.persistence import review as repository +from packages.persistence.security import begin_idempotency, complete_idempotency, utc_now +from packages.persistence.transaction import lock_tenant + + +class ReviewApplication: + def __init__(self, identity): + self.identity = identity + + def read( + self, + kind, + token, + request_id, + *, + resource_id=None, + laboratory_id=None, + page=1, + page_size=20, + current_only=True, + status=None, + severity=None, + ): + principal = self.identity._preflight(token) + self.identity.limits.user(principal, write=False) + return self._read_once( + kind, + token, + request_id, + resource_id=resource_id, + laboratory_id=laboratory_id, + page=page, + page_size=page_size, + current_only=current_only, + status=status, + severity=severity, + ) + + @retry_deadlocks + def _read_once( + self, + kind, + token, + request_id, + *, + resource_id, + laboratory_id, + page, + page_size, + current_only, + status, + severity, + ): + with self.identity.transaction() as connection: + actor = self.identity.sessions.authenticate(connection, token).principal + if kind == "facts": + data = repository.current_facts(connection, actor, resource_id) + elif kind == "finding": + data = repository.get_finding(connection, actor, resource_id) + elif kind == "findings": + data = repository.list_findings( + connection, + actor, + page, + page_size, + laboratory_id=laboratory_id, + current_only=current_only, + status=status, + severity=severity, + ) + else: + raise ServiceError("VALIDATION_ERROR", 422, "Unknown review query") + return {"data": data, "request_id": request_id} + + def write(self, operation, token, csrf, key, resource_id, body, request_id): + principal = self.identity._preflight(token, csrf=csrf) + self.identity.limits.user(principal, write=True) + try: + resource_id = str(UUID(resource_id)) + except (TypeError, ValueError): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid resource id") from None + if operation not in {"facts", "confirm", "reject", "cannot-determine"}: + raise ServiceError("VALIDATION_ERROR", 422, "Unknown review command") + if not isinstance(body, dict): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid review command body") + return self._write_once(operation, token, csrf, key, resource_id, body, request_id) + + @retry_deadlocks + def _write_once(self, operation, token, csrf, key, resource_id, body, request_id): + path = ( + f"/api/v1/inspection-items/{resource_id}/facts" + if operation == "facts" + else f"/api/v1/findings/{resource_id}/{operation}" + ) + with self.identity.transaction() as connection: + tenant, actor_id = self.identity.sessions.locate(connection, token) + lock_tenant(connection, tenant) + owner = str(uuid4()) + claim = begin_idempotency( + connection, + tenant_id=tenant, + actor_id=actor_id, + method="POST", + path=path, + key=key, + request_hash=self.identity.sessions.request_digest(body), + expires_at=utc_now() + timedelta(hours=24), + lease_owner=owner, + ) + session = self.identity.sessions.authenticate(connection, token) + self.identity.sessions.require_csrf(session, csrf) + actor = session.principal + if claim.state == "completed": + if operation == "facts": + repository.authorize_item(connection, actor, resource_id) + else: + repository.authorize_finding(connection, actor, resource_id) + return claim.response_status, claim.response + if operation == "facts": + data = repository.edit_facts(connection, actor, resource_id, body, request_id) + status = 202 + else: + data = repository.decide_finding( + connection, actor, resource_id, body, operation, request_id + ) + status = 200 + response = {"data": data, "request_id": request_id} + complete_idempotency( + connection, claim, status=status, response=response, lease_owner=owner + ) + return status, response diff --git a/packages/application/rule_execution.py b/packages/application/rule_execution.py new file mode 100644 index 0000000..d6d0d66 --- /dev/null +++ b/packages/application/rule_execution.py @@ -0,0 +1,105 @@ +"""Rule evaluation worker orchestration.""" + +from threading import Event, Thread +from uuid import uuid4 + +from packages.application.dispatch import transaction +from packages.domain.rule_execution import RuleLeaseLost +from packages.persistence import rule_execution +from packages.rules.evaluator import RuleDefinitionError, evaluate_bundle +from packages.shared.environment import development_environment + + +class RuleHeartbeat: + interval = 10 + + def __init__(self, execution, lease, cancel): + self.execution, self.lease, self.cancel = execution, lease, cancel + self.stop = Event() + self.lost = Event() + self.thread = Thread(target=self._loop, name="rule-lease-heartbeat", daemon=True) + + def _loop(self): + while not self.stop.wait(self.interval): + try: + renewed = self.execution.heartbeat(self.lease) + except Exception: + renewed = False + if renewed: + continue + self.lost.set() + try: + self.cancel() + except Exception: + pass + return + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, exc_type, exc, traceback): + self.stop.set() + self.thread.join(timeout=2) + if self.thread.is_alive() or self.lost.is_set(): + raise RuleLeaseLost() from None + return False + + +class RuleExecution: + def __init__(self, engine): + development_environment() + self.engine = engine + + def claim(self, message): + with transaction(self.engine) as connection: + return rule_execution.claim(connection, message, str(uuid4())) + + def heartbeat(self, lease): + with transaction(self.engine) as connection: + return rule_execution.heartbeat(connection, lease) + + def fail(self, lease, code): + with transaction(self.engine) as connection: + return rule_execution.fail(connection, lease, code) + + def commit(self, lease): + # Rule evaluation is pure over the immutable lease snapshot. Keep it + # outside the database transaction; the persistence layer re-locks the + # current item/evaluation/task and fences the write before applying it. + try: + results = evaluate_bundle( + lease.input.bundle, + lease.input.facts, + lease.input.reference_date, + tenant_id=lease.input.tenant_id, + laboratory_id=lease.input.laboratory_id, + ) + except RuleDefinitionError: + with transaction(self.engine) as connection: + return rule_execution.commit_result(connection, lease, error_code="RULESET_INVALID") + with transaction(self.engine) as connection: + return rule_execution.commit_result(connection, lease, results=results) + + def recover_expired(self, limit=100): + with transaction(self.engine) as connection: + candidates = rule_execution.expired_candidates(connection, limit) + recovered = 0 + for candidate in candidates: + with transaction(self.engine) as connection: + recovered += int(rule_execution.recover(connection, candidate)) + return recovered + + def execute(self, message): + lease = self.claim(message) + if lease is None: + return False + try: + with RuleHeartbeat(self, lease, lambda: None): + self.commit(lease) + except RuleLeaseLost: + raise + except Exception: + self.fail(lease, "INTERNAL_ERROR") + raise + return True diff --git a/packages/application/rules.py b/packages/application/rules.py new file mode 100644 index 0000000..9f23b23 --- /dev/null +++ b/packages/application/rules.py @@ -0,0 +1,154 @@ +"""Application boundary for tenant rule configuration and publication.""" + +from datetime import timedelta +from uuid import uuid4 + +from sqlalchemy.exc import IntegrityError + +from packages.application.identity import retry_deadlocks +from packages.domain.security import ServiceError +from packages.persistence.rules import RuleRepository +from packages.persistence.security import begin_idempotency, complete_idempotency, utc_now +from packages.persistence.transaction import lock_tenant + + +class RuleApplication: + def __init__(self, identity): + self.identity = identity + self.rules = RuleRepository() + + def read( + self, + kind, + token, + request_id, + *, + resource_id=None, + laboratory_id=None, + page=1, + page_size=20, + ): + principal = self.identity._preflight(token) + self.identity.limits.user(principal, write=False) + return self._read_once( + kind, + token, + request_id, + resource_id=resource_id, + laboratory_id=laboratory_id, + page=page, + page_size=page_size, + ) + + @retry_deadlocks + def _read_once( + self, + kind, + token, + request_id, + *, + resource_id, + laboratory_id, + page, + page_size, + ): + with self.identity.transaction() as connection: + actor = self.identity.sessions.authenticate(connection, token).principal + if kind == "rule_set": + data = ( + self.rules.get_set(connection, actor, resource_id) + if resource_id + else self.rules.list_sets(connection, actor, page, page_size, laboratory_id) + ) + elif kind == "rule_version": + data = ( + self.rules.get_version(connection, actor, resource_id) + if resource_id + else self.rules.list_versions(connection, actor, page, page_size, laboratory_id) + ) + elif kind == "rule_bundle": + data = self.rules.list_bundles(connection, actor, page, page_size, laboratory_id) + else: + raise ValueError("Unknown rule resource") + return {"data": data, "request_id": request_id} + + def write(self, kind, operation, token, csrf, key, body, request_id, *, resource_id=None): + principal = self.identity._preflight(token, csrf=csrf) + self.identity.limits.user(principal, write=True) + try: + return self._write_once( + kind, + operation, + token, + csrf, + key, + body, + request_id, + resource_id=resource_id, + ) + except IntegrityError as error: + if getattr(error.orig, "args", [None])[0] == 1062: + raise ServiceError("STATE_CONFLICT", 409, "Resource already exists") from None + raise + + @retry_deadlocks + def _write_once(self, kind, operation, token, csrf, key, body, request_id, *, resource_id): + paths = { + "rule_set": "/api/v1/rule-sets", + "rule_version": "/api/v1/rule-versions", + "rule_bundle": "/api/v1/rule-bundles", + } + path = paths[kind] + if resource_id is not None: + path += f"/{resource_id}/{operation}" + with self.identity.transaction() as connection: + tenant, user = self.identity.sessions.locate(connection, token) + lock_tenant(connection, tenant) + owner = str(uuid4()) + claim = begin_idempotency( + connection, + tenant_id=tenant, + actor_id=user, + method="POST", + path=path, + key=key, + request_hash=self.identity.sessions.request_digest(body), + expires_at=utc_now() + timedelta(hours=24), + lease_owner=owner, + ) + session = self.identity.sessions.authenticate(connection, token) + self.identity.sessions.require_csrf(session, csrf) + actor = session.principal + if claim.state == "completed": + if kind == "rule_set": + self.rules.get_set(connection, actor, claim.response["data"]["id"], share=True) + elif kind == "rule_version": + self.rules.get_version( + connection, actor, claim.response["data"]["id"], share=True + ) + else: + # Bundle create is idempotent by checksum and the cached + # projection is tenant-scoped by the authenticated actor. + pass + return claim.response_status, claim.response + if kind == "rule_set": + data = self.rules.create_set(connection, actor, body, request_id) + status = 201 + elif kind == "rule_version" and operation == "import": + data = self.rules.import_version(connection, actor, body, request_id) + status = 201 + elif kind == "rule_version": + data = self.rules.transition( + connection, actor, resource_id, operation, body, request_id + ) + status = 200 + elif kind == "rule_bundle": + data = self.rules.create_bundle(connection, actor, body, request_id) + status = 201 + else: + raise ValueError("Unknown rule command") + response = {"data": data, "request_id": request_id} + complete_idempotency( + connection, claim, status=status, response=response, lease_owner=owner + ) + return status, response diff --git a/packages/domain/image_download.py b/packages/domain/image_download.py new file mode 100644 index 0000000..1815b52 --- /dev/null +++ b/packages/domain/image_download.py @@ -0,0 +1,75 @@ +"""Authorized, fixed-version O/A download references; never staging or arbitrary keys.""" + +from dataclasses import dataclass + +from packages.domain.image_validation import EXTENSIONS +from packages.domain.security import Permission, ServiceError, authorize, not_found +from packages.domain.uploads import digest, identifier + +DOWNLOAD_SECONDS = 60 + + +@dataclass(frozen=True) +class ImageDownload: + tenant_id: str + laboratory_id: str + image_id: str + variant: str + sha256: str + mime_type: str + key: str + object_version: str + + +def validate_reference(reference): + if not isinstance(reference, ImageDownload): + raise ValueError("Invalid image download reference") + for value in (reference.tenant_id, reference.laboratory_id, reference.image_id): + identifier(value) + digest(reference.sha256) + if reference.variant not in {"analysis", "original"} or reference.mime_type not in EXTENSIONS: + raise ValueError("Invalid image download variant or media type") + if reference.variant == "analysis" and reference.mime_type != "image/png": + raise ValueError("Analysis must be PNG") + root = f"tenant/{reference.tenant_id}/lab/{reference.laboratory_id}" + expected = ( + f"{root}/{reference.variant}/{reference.image_id}/" + f"{reference.sha256}.{EXTENSIONS[reference.mime_type]}" + ) + if reference.key != expected: + raise ValueError("Invalid image download key") + version = reference.object_version + if ( + not isinstance(version, str) + or not 1 <= len(version) <= 200 + or version == "null" + or any(not 33 <= ord(c) <= 126 for c in version) + ): + raise ValueError("Invalid image download version") + + +def download_reference(actor, row, variant): + if variant not in {"analysis", "original"}: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid download variant") + if row is None or row["tenant_id"] != actor.tenant_id: + raise not_found() + authorize(actor, Permission.READ, row["laboratory_id"]) + if variant == "original": + authorize(actor, Permission.ADMIN) + if row["status"] != "ready": + raise ServiceError("STATE_CONFLICT", 409, "Image is not ready for download") + reference = ImageDownload( + row["tenant_id"], + row["laboratory_id"], + row["id"], + variant, + row[f"{variant}_sha256"], + "image/png" if variant == "analysis" else row["mime_type"], + row[f"{variant}_key"], + row[f"{variant}_object_version"], + ) + try: + validate_reference(reference) + except (ValueError, TypeError): + raise ServiceError("STATE_CONFLICT", 409, "Image download evidence is incomplete") from None + return reference diff --git a/packages/domain/image_validation.py b/packages/domain/image_validation.py new file mode 100644 index 0000000..b0fe2a0 --- /dev/null +++ b/packages/domain/image_validation.py @@ -0,0 +1,75 @@ +"""Trusted validator outcomes and deterministic object references.""" + +from dataclasses import dataclass + +from packages.domain.uploads import digest, identifier + +CONTENT_ERRORS = frozenset( + { + "OBJECT_NOT_FOUND", + "IMAGE_INVALID", + "IMAGE_TOO_LARGE", + "UNSUPPORTED_MEDIA_TYPE", + "HASH_MISMATCH", + } +) +EXTENSIONS = {"image/jpeg": "jpg", "image/png": "png", "image/webp": "webp"} + + +def image_keys(tenant, input, analysis_sha): + for value in (tenant, input.laboratory_id, input.image_id): + identifier(value) + digest(input.expected_sha256) + digest(analysis_sha) + root = f"tenant/{tenant}/lab/{input.laboratory_id}" + return ( + f"{root}/original/{input.image_id}/{input.expected_sha256}.{EXTENSIONS[input.mime_type]}", + f"{root}/analysis/{input.image_id}/{analysis_sha}.png", + ) + + +@dataclass(frozen=True) +class ValidatedImage: + original_key: str + original_version: str + analysis_key: str + analysis_version: str + analysis_sha256: str + width: int + height: int + + +@dataclass(frozen=True) +class RejectedImage: + code: str + + def __post_init__(self): + if self.code not in CONTENT_ERRORS: + raise ValueError("Not an image content error") + + +def validate_result(tenant, input, result): + if not isinstance(result, ValidatedImage): + raise ValueError("Invalid ready image outcome") + digest(result.analysis_sha256) + if image_keys(tenant, input, result.analysis_sha256) != ( + result.original_key, + result.analysis_key, + ): + raise ValueError("Invalid image result object keys") + for version in (result.original_version, result.analysis_version): + if ( + not isinstance(version, str) + or not 1 <= len(version) <= 200 + or version == "null" + or any(not 33 <= ord(c) <= 126 for c in version) + ): + raise ValueError("Invalid image result object version") + if ( + type(result.width) is not int + or type(result.height) is not int + or not 1 <= result.width <= 10000 + or not 1 <= result.height <= 10000 + or result.width * result.height > 40_000_000 + ): + raise ValueError("Invalid image result dimensions") diff --git a/packages/domain/inference_execution.py b/packages/domain/inference_execution.py new file mode 100644 index 0000000..e0c49cb --- /dev/null +++ b/packages/domain/inference_execution.py @@ -0,0 +1,151 @@ +"""Pure values and validation rules for inference pipeline execution.""" + +from dataclasses import dataclass +from datetime import datetime + +from packages.inference_protocol.contract import validate +from packages.inference_protocol.hashing import request_hash + + +class InferenceLeaseLost(RuntimeError): + def __init__(self): + super().__init__("LEASE_LOST") + + +class InferenceInvalidResult(ValueError): + def __init__(self): + super().__init__("SCHEMA_MISMATCH") + + +# Codes which may safely return the task to retry_wait. Version, hash and +# schema errors are deliberately absent: retrying the same invalid input +# cannot make it valid. +RETRYABLE_ERRORS = frozenset( + { + "MODEL_NOT_READY", + "AI_BUSY", + "AI_TIMEOUT", + "MODEL_OOM", + "DEPENDENCY_UNAVAILABLE", + "STAGE_TIMEOUT", + } +) +# Compatibility shape with the image execution domain: callers can classify a +# protocol error without knowing which codes are retryable. +TECHNICAL_ERRORS = { + code: code in RETRYABLE_ERRORS + for code in ( + *RETRYABLE_ERRORS, + "MODEL_ERROR", + "SCHEMA_MISMATCH", + "HASH_MISMATCH", + "MODEL_VERSION_UNAVAILABLE", + "INTERNAL_ERROR", + ) +} + + +@dataclass(frozen=True) +class InferenceImage: + image_id: str + object_key: str + sha256: str + mime_type: str + role: str + location_id: str + parent_image_id: str | None + + +@dataclass(frozen=True) +class InferenceInput: + tenant_id: str + laboratory_id: str + item_id: str + run_id: str + submission_revision: int + model_bundle_id: str + model_checksum: str + dictionary_version_id: str + dictionary_sha256: str + pipeline_version: str + device_profile: str + images: tuple[InferenceImage, ...] + + @property + def request_hash(self) -> str: + return request_hash( + { + "tenant_id": self.tenant_id, + "laboratory_id": self.laboratory_id, + "item_id": self.item_id, + "run_id": self.run_id, + "submission_revision": self.submission_revision, + "model_bundle_id": self.model_bundle_id, + "model_checksum": self.model_checksum, + "dictionary_version_id": self.dictionary_version_id, + "dictionary_sha256": self.dictionary_sha256, + "pipeline_version": self.pipeline_version, + "device_profile": self.device_profile, + "image_refs": [image.__dict__ for image in self.images], + } + ) + + +@dataclass(frozen=True) +class InferenceLease: + tenant_id: str + task_id: str + owner: str + attempt_id: str + token: int + generation: int + attempt: int + input: InferenceInput + + +def request_payload(lease: InferenceLease, deadline_at: datetime) -> dict: + return { + "run_id": lease.input.run_id, + "attempt_id": lease.attempt_id, + "fencing_token": lease.token, + "tenant_id": lease.input.tenant_id, + "laboratory_id": lease.input.laboratory_id, + "item_id": lease.input.item_id, + "submission_revision": lease.input.submission_revision, + "model_bundle_id": lease.input.model_bundle_id, + "model_checksum": lease.input.model_checksum, + "dictionary_version_id": lease.input.dictionary_version_id, + "dictionary_sha256": lease.input.dictionary_sha256, + "pipeline_version": lease.input.pipeline_version, + "device_profile": lease.input.device_profile, + "deadline_at": deadline_at.isoformat().replace("+00:00", "Z"), + "request_hash": lease.input.request_hash, + "image_refs": [image.__dict__ for image in lease.input.images], + } + + +def validate_result(result: dict, lease: InferenceLease) -> None: + try: + validate("InferenceResult", result) + except ValueError as exc: + raise InferenceInvalidResult() from exc + expected = { + "run_id": lease.input.run_id, + "attempt_id": lease.attempt_id, + "fencing_token": lease.token, + "tenant_id": lease.input.tenant_id, + "request_hash": lease.input.request_hash, + "model_bundle_id": lease.input.model_bundle_id, + "model_checksum": lease.input.model_checksum, + "dictionary_version_id": lease.input.dictionary_version_id, + "dictionary_sha256": lease.input.dictionary_sha256, + "pipeline_version": lease.input.pipeline_version, + } + if any(result.get(key) != value for key, value in expected.items()): + raise InferenceInvalidResult() + hashes = result.get("input_hashes") + expected_hashes = [ + {"image_id": image.image_id, "sha256": image.sha256} for image in lease.input.images + ] + if hashes != expected_hashes: + raise InferenceInvalidResult() diff --git a/packages/domain/job_execution.py b/packages/domain/job_execution.py new file mode 100644 index 0000000..dfbbe51 --- /dev/null +++ b/packages/domain/job_execution.py @@ -0,0 +1,57 @@ +"""Execution values for validate_image; no queue or database dependencies.""" + +import random +from dataclasses import dataclass +from datetime import timedelta + + +class LeaseLost(RuntimeError): + def __init__(self): + super().__init__("LEASE_LOST") + + +class InvalidDispatch(ValueError): + def __init__(self): + super().__init__("Invalid or future task dispatch") + + +# This subset handles infrastructure failures, not image-validation outcomes. +# Content errors must atomically produce rejected image/upload via the handler. +TECHNICAL_ERRORS = {"DEPENDENCY_UNAVAILABLE": True, "STAGE_TIMEOUT": True, "INTERNAL_ERROR": False} + + +def retry_delay(attempt, *, uniform=random.uniform): + if type(attempt) is not int or attempt not in (1, 2, 3): + raise ValueError("Retry delay requires attempt 1..3") + seconds = (5, 30, 120)[attempt - 1] + jitter = uniform(0, 0.2) + if not 0 <= jitter <= 0.2: + raise ValueError("Invalid jitter") + return timedelta(milliseconds=int(seconds * (1 + jitter) * 1000)) + + +@dataclass(frozen=True) +class ImageInput: + image_id: str + upload_id: str + laboratory_id: str + owner_type: str + owner_id: str + key: str + object_version: str + expected_sha256: str + size_bytes: int + mime_type: str + image_version: int + upload_version: int + + +@dataclass(frozen=True) +class ImageLease: + tenant_id: str + task_id: str + owner: str + token: int + generation: int + attempt: int + input: ImageInput diff --git a/packages/domain/job_replay.py b/packages/domain/job_replay.py new file mode 100644 index 0000000..0a07bf8 --- /dev/null +++ b/packages/domain/job_replay.py @@ -0,0 +1,32 @@ +"""Manual validate_image replay guards; no queue or object calls.""" + +from packages.domain.security import Permission, ServiceError, authorize, not_found, require_version + + +def replay_body(body): + if ( + not isinstance(body, dict) + or set(body) != {"expected_version", "reason"} + or not isinstance(body["reason"], str) + or not 1 <= len(body["reason"]) <= 2000 + or not body["reason"].strip() + ): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid replay command") + require_version(body["expected_version"], body["expected_version"]) + + +def require_replay(actor, task, current, expected_version): + authorize(actor, Permission.ADMIN) + if ( + task is None + or task["tenant_id"] != actor.tenant_id + or task["task_type"] != "validate_image" + ): + raise not_found() + require_version(task["version"], expected_version) + if task["state"] not in {"dead_letter", "failed"} or current is None: + raise ServiceError("STATE_CONFLICT", 409, "Task source is not replayable") + if any( + task[name] >= 2147483647 for name in ("replay_generation", "dispatch_sequence", "version") + ): + raise ServiceError("STATE_CONFLICT", 409, "Task replay counter exhausted") diff --git a/packages/domain/rule_execution.py b/packages/domain/rule_execution.py new file mode 100644 index 0000000..4ec4e2b --- /dev/null +++ b/packages/domain/rule_execution.py @@ -0,0 +1,38 @@ +"""Values and deterministic error policy for rule_evaluation tasks.""" + +from dataclasses import dataclass + + +class RuleLeaseLost(RuntimeError): + def __init__(self): + super().__init__("LEASE_LOST") + + +RULE_RETRYABLE = frozenset({"DEPENDENCY_UNAVAILABLE", "STAGE_TIMEOUT", "INTERNAL_ERROR"}) + + +@dataclass(frozen=True) +class RuleInput: + tenant_id: str + item_id: str + run_id: str + fact_revision_id: str + evaluation_id: str + rule_bundle_id: str + reference_date: str + facts: dict + bundle: dict + laboratory_id: str | None = None + bundle_checksum: str | None = None + + +@dataclass(frozen=True) +class RuleLease: + tenant_id: str + task_id: str + owner: str + attempt_id: str + token: int + generation: int + attempt: int + input: RuleInput diff --git a/packages/persistence/dispatch.py b/packages/persistence/dispatch.py new file mode 100644 index 0000000..0055cd2 --- /dev/null +++ b/packages/persistence/dispatch.py @@ -0,0 +1,269 @@ +"""Short MySQL transactions for I-03A1 validate_image delivery only. + +No broker I/O belongs here. Task execution leases and domain transitions are +deliberately separate from publisher leases. +Callers use READ COMMITTED scheduler connections, not the API's RR snapshots. +""" + +import json +from datetime import timedelta +from uuid import uuid4 + +from sqlalchemy import text + +from packages.domain.uploads import capture_time, identifier +from packages.persistence.transaction import require_transaction +from packages.persistence.users import timestamp +from packages.shared.json_hash import canonical_json + + +def validate_dispatch(message): + """Fail closed on the implemented task-message-v1 branches (schema 1.1).""" + import re + + fields = { + "schema_version", + "tenant_id", + "trace_id", + "task_id", + "task_type", + "resource_id", + "replay_generation", + "dispatch_sequence", + "created_at", + "payload", + } + if not isinstance(message, dict) or set(message) != fields: + raise ValueError("Invalid dispatch envelope") + if message["schema_version"] != "1.1" or message["task_type"] not in { + "validate_image", + "inference_pipeline", + "rule_evaluation", + }: + raise ValueError("Unsupported dispatch protocol or task type") + for name in ("tenant_id", "task_id", "resource_id"): + identifier(message[name]) + trace = message["trace_id"] + if not isinstance(trace, str) or re.fullmatch("[a-f0-9]{32}", trace) is None: + raise ValueError("Invalid trace") + for name, lower in (("replay_generation", 0), ("dispatch_sequence", 1)): + value = message[name] + if type(value) is not int or not lower <= value <= 2147483647: + raise ValueError("Invalid dispatch counter") + capture_time(message["created_at"]) + payload = message["payload"] + if not isinstance(payload, dict): + raise ValueError("Invalid task payload") + if message["task_type"] == "validate_image": + if set(payload) != {"upload_id", "image_id"}: + raise ValueError("Invalid validate_image payload") + identifier(payload["upload_id"]) + identifier(payload["image_id"]) + if payload["image_id"] != message["resource_id"]: + raise ValueError("Inconsistent dispatch resource") + elif message["task_type"] == "inference_pipeline": + if set(payload) != { + "run_id", + "submission_revision", + "model_bundle_id", + "dictionary_version_id", + "rule_bundle_id", + }: + raise ValueError("Invalid inference_run payload") + for name in ("run_id", "model_bundle_id", "dictionary_version_id", "rule_bundle_id"): + identifier(payload[name]) + if payload["run_id"] != message["resource_id"]: + raise ValueError("Inconsistent inference resource") + if ( + type(payload["submission_revision"]) is not int + or not 1 <= payload["submission_revision"] <= 2147483647 + ): + raise ValueError("Invalid inference revision") + else: + if set(payload) != {"run_id", "fact_revision_id", "rule_bundle_id"}: + raise ValueError("Invalid rule_evaluation payload") + for name in ("run_id", "fact_revision_id", "rule_bundle_id"): + identifier(payload[name]) + if payload["run_id"] != message["resource_id"]: + raise ValueError("Inconsistent rule evaluation resource") + return message + + +def decoded(value): + return json.loads(value) if isinstance(value, str) else value + + +def db_now(connection): + require_transaction(connection) + return connection.scalar(text("SELECT UTC_TIMESTAMP(3)")) + + +def claim(connection, owner): + """Claim just one row so waiting behind slow publishes cannot expire a batch.""" + identifier(owner) + db_now(connection) + row = ( + connection.execute( + text( + "SELECT * FROM outbox_events WHERE event_type='TaskDispatch' " + "AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.task_type')) " + "IN ('validate_image','inference_pipeline','rule_evaluation') " + "AND state='pending' AND available_at<=UTC_TIMESTAMP(3) " + "ORDER BY available_at,id LIMIT 1 FOR UPDATE SKIP LOCKED" + ) + ) + .mappings() + .first() + ) + if row is None: + return None + now = db_now(connection) + connection.execute( + text( + "UPDATE outbox_events SET state='leased',lease_owner=:owner,lease_until=:until," + "attempts=attempts+1,updated_at=:now,version=version+1 WHERE id=:id" + ), + {"owner": owner, "until": now + timedelta(seconds=30), "now": now, "id": row["id"]}, + ) + return dict(row) + + +def owned_update(connection, row, owner, operation): + """Lock before reading the database clock, preventing stale-time acceptance.""" + require_transaction(connection) + current = ( + connection.execute( + text( + "SELECT state,lease_owner,lease_until FROM outbox_events " + "WHERE tenant_id=:tenant AND id=:id FOR UPDATE" + ), + {"tenant": row["tenant_id"], "id": row["id"]}, + ) + .mappings() + .first() + ) + now = db_now(connection) + if ( + current is None + or current["state"] != "leased" + or current["lease_owner"] != owner + or current["lease_until"] is None + or current["lease_until"] <= now + ): + return False + updates = { + "renew": "lease_until=:until", + "published": "state='published',published_at=:now,lease_owner=NULL,lease_until=NULL", + "release": "state='pending',available_at=:retry,lease_owner=NULL,lease_until=NULL", + } + connection.execute( + text( + f"UPDATE outbox_events SET {updates[operation]},updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "tenant": row["tenant_id"], + "id": row["id"], + "now": now, + "until": now + timedelta(seconds=30), + "retry": now + timedelta(seconds=30), + }, + ) + return True + + +def recover_publications(connection, limit=100): + require_transaction(connection) + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("Batch limit must be 1..100") + rows = ( + connection.execute( + text( + "SELECT id FROM outbox_events WHERE event_type='TaskDispatch' " + "AND JSON_UNQUOTE(JSON_EXTRACT(payload,'$.task_type')) " + "IN ('validate_image','inference_pipeline','rule_evaluation') " + "AND state='leased' AND lease_until<=UTC_TIMESTAMP(3) " + "ORDER BY lease_until,id LIMIT :limit FOR UPDATE SKIP LOCKED" + ), + {"limit": limit}, + ) + .mappings() + .all() + ) + now = db_now(connection) + for row in rows: + connection.execute( + text( + "UPDATE outbox_events SET state='pending',lease_owner=NULL,lease_until=NULL," + "available_at=:now,updated_at=:now,version=version+1 WHERE id=:id" + ), + {"id": row["id"], "now": now}, + ) + return len(rows) + + +def redispatch(connection, limit=100): + """Rebuild notices from DB payload, even if the initial event was published. + + last_dispatched_at is the time an intent was registered, not a broker ack. + It is changed atomically with the sequence and new outbox row. + """ + require_transaction(connection) + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("Batch limit must be 1..100") + rows = ( + connection.execute( + text( + "SELECT * FROM task_runs WHERE task_type IN " + "('validate_image','inference_pipeline','rule_evaluation') " + "AND state IN ('ready','retry_wait') AND available_at<=UTC_TIMESTAMP(3) " + "AND (last_dispatched_at IS NULL OR " + "last_dispatched_at<=UTC_TIMESTAMP(3)-INTERVAL 30 SECOND) " + "ORDER BY available_at,id LIMIT :limit FOR UPDATE SKIP LOCKED" + ), + {"limit": limit}, + ) + .mappings() + .all() + ) + now = db_now(connection) + for row in rows: + sequence = row["dispatch_sequence"] + 1 + message = validate_dispatch( + { + "schema_version": "1.1", + "tenant_id": row["tenant_id"], + "trace_id": uuid4().hex, + "task_id": row["id"], + "task_type": row["task_type"], + "resource_id": row["resource_id"], + "replay_generation": row["replay_generation"], + "dispatch_sequence": sequence, + "created_at": timestamp(now), + "payload": decoded(row["payload"]), + } + ) + connection.execute( + text( + "UPDATE task_runs SET dispatch_sequence=:sequence,last_dispatched_at=:now," + "updated_at=:now,version=version+1 WHERE tenant_id=:tenant AND id=:id" + ), + {"sequence": sequence, "now": now, "tenant": row["tenant_id"], "id": row["id"]}, + ) + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) VALUES " + "(:id,:tenant,'TaskDispatch','task_run',:task,:version,:payload,'pending'," + ":now,:now,:now)" + ), + { + "id": str(uuid4()), + "tenant": row["tenant_id"], + "task": row["id"], + "version": row["version"] + 1, + "payload": canonical_json(message), + "now": now, + }, + ) + return len(rows) diff --git a/packages/persistence/foundations.py b/packages/persistence/foundations.py index b0ed422..9d4df26 100644 --- a/packages/persistence/foundations.py +++ b/packages/persistence/foundations.py @@ -50,14 +50,25 @@ def lab_filter(actor, column, params, laboratory_id=None): return f" AND {column} IN ({','.join(names)})" -def page_query(connection, table, columns, where, params, page, page_size, *, project_row=project): +def page_query( + connection, + table, + columns, + where, + params, + page, + page_size, + *, + project_row=project, + order_by="created_at DESC,id DESC", +): # SQL fragments are fixed by repositories, never supplied by the request. total = connection.scalar(text(f"SELECT COUNT(*) FROM {table} WHERE {where}"), params) rows = ( connection.execute( text( f"SELECT {columns} FROM {table} WHERE {where} " - "ORDER BY created_at DESC,id DESC LIMIT :limit OFFSET :offset" + f"ORDER BY {order_by} LIMIT :limit OFFSET :offset" ), {**params, "limit": page_size, "offset": (page - 1) * page_size}, ) diff --git a/packages/persistence/image_validation.py b/packages/persistence/image_validation.py new file mode 100644 index 0000000..9432a8d --- /dev/null +++ b/packages/persistence/image_validation.py @@ -0,0 +1,155 @@ +"""Server-owned image outcome callback inside the execution fencing transaction.""" + +from types import SimpleNamespace +from uuid import UUID, uuid4 + +from sqlalchemy import text + +from packages.domain.image_validation import RejectedImage, ValidatedImage, validate_result +from packages.persistence.dispatch import db_now, decoded +from packages.persistence.security import append_audit +from packages.persistence.transaction import require_transaction +from packages.persistence.users import timestamp +from packages.shared.json_hash import canonical_json + + +def write_image_result(connection, tenant, task_id, input, outcome): + """Call ONLY through ImageExecution.commit/execute, after owner/input locks.""" + require_transaction(connection) + params = {"tenant": tenant, "id": input.image_id} + ready = isinstance(outcome, ValidatedImage) + if ready: + validate_result(tenant, input, outcome) + connection.execute( + text( + "UPDATE asset_images SET status='ready',original_key=:original," + "original_object_version=:original_version,analysis_key=:analysis," + "analysis_object_version=:analysis_version,analysis_sha256=:sha," + "width=:width,height=:height,version=version+1,updated_at=UTC_TIMESTAMP(3) " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + **params, + "original": outcome.original_key, + "original_version": outcome.original_version, + "analysis": outcome.analysis_key, + "analysis_version": outcome.analysis_version, + "sha": outcome.analysis_sha256, + "width": outcome.width, + "height": outcome.height, + }, + ) + elif isinstance(outcome, RejectedImage): + connection.execute( + text( + "UPDATE asset_images SET status='rejected',analysis_key=NULL," + "analysis_object_version=NULL,analysis_sha256=NULL,width=NULL,height=NULL," + "version=version+1,updated_at=UTC_TIMESTAMP(3) WHERE tenant_id=:tenant AND id=:id" + ), + params, + ) + else: + raise ValueError("Invalid image outcome") + status = "ready" if ready else "rejected" + connection.execute( + text( + "UPDATE uploads SET status=:status,version=version+1,updated_at=UTC_TIMESTAMP(3) " + "WHERE tenant_id=:tenant AND id=:id" + ), + {"tenant": tenant, "id": input.upload_id, "status": status}, + ) + if input.owner_type == "inspection_item": + owner_params = {"tenant": tenant, "id": input.owner_id} + if ready: + connection.execute( + text( + "UPDATE inspection_items SET status='uploaded',version=version+1," + "updated_at=UTC_TIMESTAMP(3) WHERE tenant_id=:tenant AND id=:id " + "AND status='draft'" + ), + owner_params, + ) + connection.execute( + text( + "UPDATE inspections SET status='in_progress',version=version+1," + "updated_at=UTC_TIMESTAMP(3) WHERE tenant_id=:tenant AND status='draft' " + "AND id=(SELECT inspection_id FROM inspection_items " + "WHERE tenant_id=:tenant AND id=:id)" + ), + owner_params, + ) + else: + # The owner mutex held by commit serializes validators and upload completion. + has_ready = connection.scalar( + text( + "SELECT id FROM asset_images WHERE tenant_id=:tenant " + "AND inspection_item_id=:id AND status='ready' LIMIT 1" + ), + owner_params, + ) + connection.execute( + text( + "UPDATE inspection_items SET status=:status,version=version+1," + "updated_at=UTC_TIMESTAMP(3) WHERE tenant_id=:tenant AND id=:id " + "AND status<>:status" + ), + {**owner_params, "status": "uploaded" if has_ready else "draft"}, + ) + # A service execution has no human actor; never impersonate the uploading user. + source = connection.scalar( + text( + "SELECT payload FROM outbox_events WHERE tenant_id=:tenant " + "AND aggregate_id=:task AND event_type='TaskDispatch' ORDER BY created_at,id LIMIT 1" + ), + {"tenant": tenant, "task": task_id}, + ) + trace = decoded(source)["trace_id"] if source else UUID(task_id).hex + append_audit( + connection, + principal=SimpleNamespace(tenant_id=tenant, user_id=None), + action="image.validate", + resource_type="image", + resource_id=input.image_id, + changes={ + "status": status, + "error_code": None if ready else outcome.code, + "owner_type": input.owner_type, + "owner_id": input.owner_id, + }, + reason="Validate fixed-version upload", + request_id=str(UUID(trace)), + ) + if ready: + now, event_id = db_now(connection), str(uuid4()) + version = input.image_version + 1 + event = { + "schema_version": "1.1", + "tenant_id": tenant, + "trace_id": trace, + "event_id": event_id, + "event_type": "ImageValidated", + "aggregate_type": "image", + "aggregate_id": input.image_id, + "aggregate_version": version, + "occurred_at": timestamp(now), + "payload": { + "image_id": input.image_id, + "owner_id": input.owner_id, + "owner_type": input.owner_type, + "analysis_sha256": outcome.analysis_sha256, + }, + } + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) VALUES " + "(:event,:tenant,'ImageValidated','image',:id,:version,:payload,'pending',:now,:now,:now)" + ), + { + **params, + "event": event_id, + "version": version, + "payload": canonical_json(event), + "now": now, + }, + ) diff --git a/packages/persistence/images.py b/packages/persistence/images.py index 8644d2b..5e90974 100644 --- a/packages/persistence/images.py +++ b/packages/persistence/images.py @@ -24,6 +24,18 @@ class ImageRepository: + def download_row(self, connection, actor, image_id): + # Locking current read: a concurrent deletion/input update cannot race signing. + # User/role/session locks from authenticate are already held by the caller. + return ( + connection.execute( + text("SELECT * FROM asset_images WHERE tenant_id=:tenant AND id=:id FOR SHARE"), + {"tenant": actor.tenant_id, "id": image_id}, + ) + .mappings() + .first() + ) + @staticmethod def project(row): data = project({name: row[name] for name in PUBLIC_FIELDS}) diff --git a/packages/persistence/inference_execution.py b/packages/persistence/inference_execution.py new file mode 100644 index 0000000..d0ac0e0 --- /dev/null +++ b/packages/persistence/inference_execution.py @@ -0,0 +1,616 @@ +"""Inference task lease and result persistence. + +The lock order is item -> current run -> task. AI calls happen outside these +short transactions; only the final fenced write is performed while locked. +""" + +from datetime import timedelta +from uuid import uuid4 + +from sqlalchemy import text + +from packages.domain.inference_execution import ( + RETRYABLE_ERRORS, + InferenceImage, + InferenceInput, + InferenceInvalidResult, + InferenceLease, + InferenceLeaseLost, + validate_result, +) +from packages.domain.job_execution import retry_delay +from packages.persistence.dispatch import db_now, decoded, validate_dispatch +from packages.persistence.transaction import require_transaction +from packages.persistence.users import timestamp +from packages.shared.json_hash import canonical_json, sha256_json + + +def _task(connection, tenant, task_id, *, lock=False): + return ( + connection.execute( + text( + "SELECT * FROM task_runs WHERE tenant_id=:tenant AND id=:id" + + (" FOR UPDATE" if lock else "") + ), + {"tenant": tenant, "id": task_id}, + ) + .mappings() + .first() + ) + + +def _load(connection, tenant, task_id): + require_transaction(connection) + task = _task(connection, tenant, task_id) + if task is None or task["task_type"] != "inference_pipeline": + return None, None + tenant_status = connection.scalar( + text("SELECT status FROM tenants WHERE id=:tenant FOR SHARE"), {"tenant": tenant} + ) + if tenant_status != "active": + return task, None + # Domain-first lock order: tenant -> item -> current run -> task. + run = ( + connection.execute( + text("SELECT * FROM inference_runs WHERE tenant_id=:tenant AND id=:run"), + {"tenant": tenant, "run": task["resource_id"]}, + ) + .mappings() + .first() + ) + if run is None: + return None, None + item = ( + connection.execute( + text("SELECT * FROM inspection_items WHERE tenant_id=:tenant AND id=:item FOR UPDATE"), + {"tenant": tenant, "item": run["item_id"]}, + ) + .mappings() + .first() + ) + run = ( + connection.execute( + text("SELECT * FROM inference_runs WHERE tenant_id=:tenant AND id=:run FOR UPDATE"), + {"tenant": tenant, "run": task["resource_id"]}, + ) + .mappings() + .first() + ) + task = _task(connection, tenant, task_id, lock=True) + if ( + item is None + or item["current_run_id"] != run["id"] + or run["status"] in {"superseded", "completed", "needs_retake", "needs_review", "failed"} + ): + return task, None + payload = decoded(task["payload"]) + model = ( + connection.execute( + text( + "SELECT m.checksum AS model_checksum,d.checksum AS dictionary_sha256 " + "FROM model_versions m JOIN dictionary_versions d " + "ON d.tenant_id=m.tenant_id AND d.id=m.dictionary_version_id " + "WHERE m.tenant_id=:tenant AND m.id=:model AND d.id=:dictionary" + ), + { + "tenant": tenant, + "model": run["model_bundle_id"], + "dictionary": run["dictionary_version_id"], + }, + ) + .mappings() + .first() + ) + image_rows = ( + connection.execute( + text( + "SELECT ri.image_id,ri.role,ri.parent_image_id,ri.analysis_sha256," + "ai.analysis_sha256 AS asset_analysis_sha256,ai.analysis_key," + "ai.analysis_object_version,ai.mime_type,ai.status AS image_status,i.location_id " + "FROM run_images ri JOIN asset_images ai ON ai.tenant_id=ri.tenant_id " + "AND ai.inspection_item_id=ri.item_id AND ai.id=ri.image_id " + "JOIN inspection_items i ON i.tenant_id=ri.tenant_id AND i.id=ri.item_id " + "WHERE ri.tenant_id=:tenant AND ri.item_id=:item AND ri.run_id=:run " + "ORDER BY ri.ordinal FOR UPDATE" + ), + {"tenant": tenant, "item": run["item_id"], "run": run["id"]}, + ) + .mappings() + .all() + ) + if model is None or not image_rows: + return task, None + if any( + row["analysis_key"] is None + or row["analysis_sha256"] != row["asset_analysis_sha256"] + or row["analysis_object_version"] in (None, "", "null") + or row["analysis_sha256"] is None + or row["mime_type"] != "image/png" + or row["image_status"] != "ready" + for row in image_rows + ): + return task, None + images = tuple( + InferenceImage( + image_id=row["image_id"], + object_key=row["analysis_key"], + sha256=row["analysis_sha256"], + mime_type=row["mime_type"], + role=row["role"], + location_id=item["location_id"], + parent_image_id=row["parent_image_id"], + ) + for row in image_rows + ) + expected = { + "run_id": run["id"], + "submission_revision": run["submission_revision"], + "model_bundle_id": run["model_bundle_id"], + "dictionary_version_id": run["dictionary_version_id"], + "rule_bundle_id": run["rule_bundle_id"], + } + if payload != expected or task["resource_id"] != run["id"]: + return task, None + return task, InferenceInput( + tenant_id=run["tenant_id"], + laboratory_id=run["laboratory_id"], + item_id=run["item_id"], + run_id=run["id"], + submission_revision=run["submission_revision"], + model_bundle_id=run["model_bundle_id"], + model_checksum=model["model_checksum"], + dictionary_version_id=run["dictionary_version_id"], + dictionary_sha256=model["dictionary_sha256"], + pipeline_version=run["pipeline_version"], + device_profile=run["device_profile"], + images=images, + ) + + +def _attempt(connection, task, attempt_id, status, code, now): + changed = connection.execute( + text( + "UPDATE task_attempts SET status=:status,error_code=:code,finished_at=:now," + "updated_at=:now,version=version+1 WHERE tenant_id=:tenant AND id=:id " + "AND status='running' AND task_id=:task AND replay_generation=:generation " + "AND attempt=:attempt AND fencing_token=:token AND lease_owner=:owner" + ), + { + "tenant": task["tenant_id"], + "id": attempt_id, + "task": task["id"], + "generation": task["replay_generation"], + "attempt": task["attempt"], + "token": task["fencing_token"], + "owner": task["lease_owner"], + "status": status, + "code": code, + "now": now, + }, + ).rowcount + if changed != 1: + raise RuntimeError("Missing current inference attempt") + + +def _owned(task, lease, now): + return ( + task is not None + and task["state"] == "leased" + and task["lease_owner"] == lease.owner + and task["fencing_token"] == lease.token + and task["replay_generation"] == lease.generation + and task["attempt"] == lease.attempt + and task["lease_until"] is not None + and task["lease_until"] > now + ) + + +def _settle(connection, task, lease, now, code, *, abandoned=False): + retry = code in RETRYABLE_ERRORS and task["attempt"] < 4 + state = "retry_wait" if retry else ("dead_letter" if code in RETRYABLE_ERRORS else "failed") + _attempt(connection, task, lease.attempt_id, "abandoned" if abandoned else "failed", code, now) + connection.execute( + text( + "UPDATE task_runs SET state=:state,last_error_code=:code,lease_owner=NULL," + "lease_until=NULL,heartbeat_at=NULL,fencing_token=fencing_token+1," + "available_at=:available,finished_at=:finished,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "tenant": task["tenant_id"], + "id": task["id"], + "state": state, + "code": code, + "available": now + retry_delay(task["attempt"]) if retry else now, + "finished": None if retry else now, + "now": now, + }, + ) + connection.execute( + text( + "UPDATE inference_runs SET status=:status,error_code=:code,updated_at=:now," + "finished_at=:finished,version=version+1 WHERE tenant_id=:tenant AND id=:run" + ), + { + "status": "retrying" if retry else "failed", + "code": code, + "finished": None if retry else now, + "now": now, + "tenant": lease.tenant_id, + "run": lease.input.run_id, + }, + ) + if not retry: + connection.execute( + text( + "UPDATE inspection_items SET status='failed',version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:item AND current_run_id=:run" + ), + { + "tenant": lease.tenant_id, + "item": lease.input.item_id, + "run": lease.input.run_id, + "now": now, + }, + ) + return state + + +def claim(connection, message, owner): + validate_dispatch(message) + task, current = _load(connection, message["tenant_id"], message["task_id"]) + if task is None: + return None + if message["replay_generation"] != task["replay_generation"]: + if message["replay_generation"] > task["replay_generation"]: + raise ValueError("future generation") + return None + if message["dispatch_sequence"] != task["dispatch_sequence"]: + if message["dispatch_sequence"] > task["dispatch_sequence"]: + raise ValueError("future sequence") + return None + if message["resource_id"] != task["resource_id"] or message["payload"] != decoded( + task["payload"] + ): + raise ValueError("Invalid inference dispatch") + now = db_now(connection) + if task["state"] not in {"ready", "retry_wait"} or task["available_at"] > now: + return None + if current is None: + connection.execute( + text( + "UPDATE task_runs SET state='failed',last_error_code='LEASE_LOST'," + "finished_at=:now,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + {"now": now, "tenant": task["tenant_id"], "id": task["id"]}, + ) + return None + if task["attempt"] >= 4: + connection.execute( + text( + "UPDATE task_runs SET state='dead_letter',last_error_code='STAGE_TIMEOUT'," + "finished_at=:now,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + {"now": now, "tenant": task["tenant_id"], "id": task["id"]}, + ) + connection.execute( + text( + "UPDATE inference_runs SET status='failed',error_code='STAGE_TIMEOUT'," + "finished_at=:now,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:run" + ), + {"now": now, "tenant": task["tenant_id"], "run": task["resource_id"]}, + ) + return None + attempt, token, attempt_id = task["attempt"] + 1, task["fencing_token"] + 1, str(uuid4()) + connection.execute( + text( + "UPDATE task_runs SET state='leased',attempt=:attempt,fencing_token=:token," + "lease_owner=:owner,lease_until=:until,heartbeat_at=:now,started_at=COALESCE(started_at,:now)," + "finished_at=NULL,last_error_code=NULL,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "attempt": attempt, + "token": token, + "owner": owner, + "until": now + timedelta(seconds=60), + "now": now, + "tenant": task["tenant_id"], + "id": task["id"], + }, + ) + connection.execute( + text( + "INSERT INTO task_attempts " + "(id,tenant_id,task_id,replay_generation,attempt,fencing_token,lease_owner," + "started_at,status,created_at,updated_at) " + "VALUES (:id,:tenant,:task,:generation,:attempt,:token,:owner,:now,'running',:now,:now)" + ), + { + "id": attempt_id, + "tenant": task["tenant_id"], + "task": task["id"], + "generation": task["replay_generation"], + "attempt": attempt, + "token": token, + "owner": owner, + "now": now, + }, + ) + connection.execute( + text( + "UPDATE inference_runs SET status='processing',stage='quality'," + "started_at=COALESCE(started_at,:now),updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:run" + ), + {"tenant": task["tenant_id"], "run": task["resource_id"], "now": now}, + ) + connection.execute( + text( + "UPDATE inspection_items SET status='quality_checking',version=version+1," + "updated_at=:now WHERE tenant_id=:tenant AND id=:item AND current_run_id=:run" + ), + {"tenant": task["tenant_id"], "item": current.item_id, "run": current.run_id, "now": now}, + ) + return InferenceLease( + task["tenant_id"], + task["id"], + owner, + attempt_id, + token, + task["replay_generation"], + attempt, + current, + ) + + +def heartbeat(connection, lease): + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or current != lease.input: + return False + connection.execute( + text( + "UPDATE task_runs SET lease_until=:until,heartbeat_at=:now,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "until": now + timedelta(seconds=60), + "now": now, + "tenant": lease.tenant_id, + "id": lease.task_id, + }, + ) + return True + + +def fail(connection, lease, code): + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or current != lease.input: + return False + _settle(connection, task, lease, now, code) + return True + + +def commit_result(connection, lease, result): + validate_result(result, lease) + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or current != lease.input: + raise InferenceLeaseLost() + outcome = result["outcome"] + if outcome == "needs_retake": + run_status, run_stage, item_status = "needs_retake", "done", "needs_retake" + elif outcome == "needs_review": + run_status, run_stage, item_status = "needs_review", "done", "needs_review" + elif outcome == "facts_ready": + run_status, run_stage, item_status = "processing", "rules", "processing" + else: + raise InferenceInvalidResult() + fact_id = None + evaluation_id = None + rule_task_id = None + if outcome == "facts_ready": + fact_id = str(uuid4()) + connection.execute( + text( + "INSERT INTO fact_revisions " + "(id,tenant_id,item_id,run_id,revision,entities,relations,dates,reason) " + "VALUES (:id,:tenant,:item,:run,1,:entities,:relations,:dates,:reason)" + ), + { + "id": fact_id, + "tenant": lease.tenant_id, + "item": lease.input.item_id, + "run": lease.input.run_id, + "entities": canonical_json(result["entities"]), + "relations": canonical_json(result["relations"]), + "dates": "[]", + "reason": "AI inference facts", + }, + ) + evaluation_id = str(uuid4()) + rule_bundle_id = connection.scalar( + text("SELECT rule_bundle_id FROM inference_runs WHERE tenant_id=:tenant AND id=:run"), + {"tenant": lease.tenant_id, "run": lease.input.run_id}, + ) + connection.execute( + text( + "INSERT INTO rule_evaluations " + "(id,tenant_id,item_id,run_id,fact_revision_id,rule_bundle_id," + "reference_date,status) " + "SELECT :id,tenant_id,item_id,id,:fact,rule_bundle_id,reference_date,'queued' " + "FROM inference_runs WHERE tenant_id=:tenant AND id=:run" + ), + { + "id": evaluation_id, + "fact": fact_id, + "tenant": lease.tenant_id, + "run": lease.input.run_id, + }, + ) + rule_task_id = str(uuid4()) + rule_payload = { + "run_id": lease.input.run_id, + "fact_revision_id": fact_id, + "rule_bundle_id": rule_bundle_id, + } + connection.execute( + text( + "INSERT INTO task_runs " + "(id,tenant_id,task_type,resource_id,logical_key,payload,state,attempt," + "replay_generation,dispatch_sequence,fencing_token,available_at,created_at," + "updated_at) " + "VALUES (:id,:tenant,'rule_evaluation',:run,:logical,:payload,'ready',0,0,1,0," + ":now,:now,:now)" + ), + { + "id": rule_task_id, + "tenant": lease.tenant_id, + "run": lease.input.run_id, + "logical": f"rule_evaluation:{evaluation_id}", + "payload": canonical_json(rule_payload), + "now": now, + }, + ) + event = { + "schema_version": "1.1", + "tenant_id": lease.tenant_id, + "trace_id": uuid4().hex, + "task_id": rule_task_id, + "task_type": "rule_evaluation", + "resource_id": lease.input.run_id, + "replay_generation": 0, + "dispatch_sequence": 1, + "created_at": timestamp(now), + "payload": rule_payload, + } + connection.execute( + text( + "INSERT INTO outbox_events " + "(id,tenant_id,event_type,aggregate_type,aggregate_id,aggregate_version," + "payload,state,available_at,created_at,updated_at) " + "VALUES (:id,:tenant,'TaskDispatch','task_run',:task,1,:payload,'pending'," + ":now,:now,:now)" + ), + { + "id": str(uuid4()), + "tenant": lease.tenant_id, + "task": rule_task_id, + "payload": canonical_json(event), + "now": now, + }, + ) + connection.execute( + text( + "UPDATE inference_runs SET status=:status,stage=:stage,result=:result," + "result_hash=:hash,finished_at=:finished,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:run" + ), + { + "status": run_status, + "stage": run_stage, + "result": canonical_json(result), + "hash": sha256_json(result), + "now": now, + "finished": now if run_stage == "done" else None, + "tenant": lease.tenant_id, + "run": lease.input.run_id, + }, + ) + connection.execute( + text( + "UPDATE inspection_items SET status=:status,current_fact_revision_id=:fact," + "current_evaluation_id=:evaluation," + "version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:item AND current_run_id=:run" + ), + { + "status": item_status, + "fact": fact_id, + "evaluation": evaluation_id, + "now": now, + "tenant": lease.tenant_id, + "item": lease.input.item_id, + "run": lease.input.run_id, + }, + ) + _attempt(connection, task, lease.attempt_id, "succeeded", None, now) + connection.execute( + text( + "UPDATE task_runs SET state='succeeded',lease_owner=NULL,lease_until=NULL," + "heartbeat_at=NULL,finished_at=:now,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + {"now": now, "tenant": lease.tenant_id, "id": lease.task_id}, + ) + + +def expired_candidates(connection, limit=100): + require_transaction(connection) + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("Batch limit must be 1..100") + return [ + dict(row) + for row in connection.execute( + text( + "SELECT tenant_id,id,fencing_token,replay_generation FROM task_runs " + "WHERE task_type='inference_pipeline' AND state='leased' " + "AND lease_until<=UTC_TIMESTAMP(3) ORDER BY lease_until,id LIMIT :limit" + ), + {"limit": limit}, + ).mappings() + ] + + +def recover(connection, candidate): + task, current = _load(connection, candidate["tenant_id"], candidate["id"]) + now = db_now(connection) + if ( + task is None + or current is None + or task["state"] != "leased" + or task["lease_until"] is None + or task["lease_until"] > now + or task["fencing_token"] != candidate["fencing_token"] + or task["replay_generation"] != candidate["replay_generation"] + ): + return False + lease = InferenceLease( + task["tenant_id"], + task["id"], + task["lease_owner"], + "", + task["fencing_token"], + task["replay_generation"], + task["attempt"], + current, + ) + attempt = connection.execute( + text( + "SELECT id FROM task_attempts WHERE tenant_id=:tenant AND task_id=:task " + "AND replay_generation=:generation AND attempt=:attempt" + ), + { + "tenant": task["tenant_id"], + "task": task["id"], + "generation": task["replay_generation"], + "attempt": task["attempt"], + }, + ).scalar_one() + lease = InferenceLease( + lease.tenant_id, + lease.task_id, + lease.owner, + attempt, + lease.token, + lease.generation, + lease.attempt, + lease.input, + ) + _settle(connection, task, lease, now, "STAGE_TIMEOUT", abandoned=True) + return True diff --git a/packages/persistence/inference_runs.py b/packages/persistence/inference_runs.py new file mode 100644 index 0000000..16c5fc8 --- /dev/null +++ b/packages/persistence/inference_runs.py @@ -0,0 +1,528 @@ +"""Atomic submitInspectionItem command persistence. + +The command loads every piece of state from MySQL under the documented lock +order, then creates the immutable inference run, run image selection, durable +task and dispatch outbox row in one transaction. No client supplied model or +tenant identifiers are trusted. +""" + +import hashlib +from uuid import uuid4 + +from sqlalchemy import text + +from packages.domain.security import Permission, authorize, not_found +from packages.domain.workflow import ( + FailedRun, + Finding, + Image, + ImageSelection, + Item, + PinnedInputs, + submit_item, +) +from packages.persistence.security import append_audit +from packages.persistence.transaction import require_transaction +from packages.persistence.users import timestamp +from packages.shared.json_hash import canonical_json + + +def _item(connection, tenant, item_id, *, lock=True): + suffix = " FOR UPDATE" if lock else "" + return ( + connection.execute( + text( + "SELECT item.id,item.version,item.tenant_id,item.laboratory_id,item.location_id," + "item.status,item.current_run_id,item.current_evaluation_id,item.submission_revision," + "inspection.status AS inspection_status,inspection.inspector_id AS creator_id " + "FROM inspection_items item JOIN inspections inspection ON " + "inspection.tenant_id=item.tenant_id AND inspection.id=item.inspection_id " + "AND inspection.laboratory_id=item.laboratory_id " + "WHERE item.tenant_id=:tenant AND item.id=:id" + suffix + ), + {"tenant": tenant, "id": item_id}, + ) + .mappings() + .first() + ) + + +def _activation(connection, tenant, laboratory): + row = ( + connection.execute( + text( + "SELECT a.id,a.model_bundle_id,a.dictionary_version_id,a.rule_bundle_id," + "a.pipeline_version,a.device_profile,m.status AS model_status," + "d.status AS dictionary_status FROM activations a " + "JOIN model_versions m ON m.tenant_id=a.tenant_id AND m.id=a.model_bundle_id " + "AND m.dictionary_version_id=a.dictionary_version_id " + "JOIN dictionary_versions d ON d.tenant_id=a.tenant_id " + "AND d.id=a.dictionary_version_id " + "WHERE a.tenant_id=:tenant AND a.laboratory_id=:lab FOR SHARE" + ), + {"tenant": tenant, "lab": laboratory}, + ) + .mappings() + .first() + ) + if row is None or row["model_status"] != "published" or row["dictionary_status"] != "published": + raise not_found() + return row + + +def _finding_rows(connection, tenant, item_id, laboratory_id): + rows = ( + connection.execute( + text( + "SELECT id,tenant_id,item_id,run_id," + "rule_evaluation_id AS evaluation_id," + "status,superseded_at FROM findings WHERE tenant_id=:tenant AND item_id=:item " + "ORDER BY id FOR UPDATE" + ), + {"tenant": tenant, "item": item_id}, + ) + .mappings() + .all() + ) + return tuple( + Finding( + id=row["id"], + tenant_id=row["tenant_id"], + laboratory_id=laboratory_id, + version=1, + item_id=row["item_id"], + run_id=row["run_id"], + evaluation_id=row["evaluation_id"], + status=row["status"], + superseded=row["superseded_at"] is not None, + evidence_ready=True, + ) + for row in rows + ) + + +def _images(connection, tenant, item_id, selections): + rows = ( + connection.execute( + text( + "SELECT id,tenant_id,laboratory_id,inspection_item_id,status,analysis_sha256 " + "FROM asset_images WHERE tenant_id=:tenant AND inspection_item_id=:item " + "ORDER BY id FOR UPDATE" + ), + {"tenant": tenant, "item": item_id}, + ) + .mappings() + .all() + ) + by_id = {row["id"]: row for row in rows} + loaded = [] + for selection in selections: + row = by_id.get(selection.image_id) + if row is None: + raise not_found() + loaded.append( + Image( + id=row["id"], + tenant_id=row["tenant_id"], + laboratory_id=row["laboratory_id"], + owner_type="inspection_item", + owner_id=item_id, + location_id=None, + status=row["status"], + available=row["analysis_sha256"] is not None, + ) + ) + return tuple(loaded), tuple(by_id[s.image_id] for s in selections) + + +def _failed_run(connection, tenant, item_id, run_id): + row = ( + connection.execute( + text( + "SELECT * FROM inference_runs WHERE tenant_id=:tenant AND item_id=:item " + "AND id=:run FOR UPDATE" + ), + {"tenant": tenant, "item": item_id, "run": run_id}, + ) + .mappings() + .first() + ) + if row is None: + raise not_found() + images = ( + connection.execute( + text( + "SELECT image_id,role,parent_image_id FROM run_images " + "WHERE tenant_id=:tenant AND item_id=:item AND run_id=:run " + "ORDER BY ordinal FOR UPDATE" + ), + {"tenant": tenant, "item": item_id, "run": run_id}, + ) + .mappings() + .all() + ) + selections = tuple( + ImageSelection( + image_id=value["image_id"], + role=value["role"], + parent_image_id=value["parent_image_id"], + ) + for value in images + ) + config = ( + connection.execute( + text( + "SELECT m.id AS model_id,m.status AS model_status,d.id AS dictionary_id," + "d.status AS dictionary_status,b.id AS rule_id " + "FROM model_versions m JOIN dictionary_versions d " + "ON d.tenant_id=m.tenant_id AND d.id=m.dictionary_version_id " + "JOIN rule_bundles b ON b.tenant_id=m.tenant_id AND b.id=:rules " + "WHERE m.tenant_id=:tenant AND m.id=:model AND d.id=:dictionary" + ), + { + "tenant": tenant, + "model": row["model_bundle_id"], + "dictionary": row["dictionary_version_id"], + "rules": row["rule_bundle_id"], + }, + ) + .mappings() + .first() + ) + available = config is not None and config["model_status"] != "retired" + pinned = PinnedInputs( + model_bundle_id=row["model_bundle_id"], + dictionary_version_id=row["dictionary_version_id"], + rule_bundle_id=row["rule_bundle_id"], + pipeline_version=row["pipeline_version"], + device_profile=row["device_profile"], + reference_date=row["reference_date"], + images=selections, + ) + return row, FailedRun( + id=row["id"], + tenant_id=row["tenant_id"], + item_id=row["item_id"], + status=row["status"], + pinned=pinned, + configuration_available=available, + ) + + +def _write_pipeline_task( + connection, *, tenant, run_id, item_id, revision, pinned, replay_of, now, trace +): + task_id = str(uuid4()) + payload = { + "run_id": run_id, + "submission_revision": revision, + "model_bundle_id": pinned.model_bundle_id, + "dictionary_version_id": pinned.dictionary_version_id, + "rule_bundle_id": pinned.rule_bundle_id, + } + connection.execute( + text( + "INSERT INTO task_runs (id,tenant_id,task_type,resource_id,logical_key,payload,state," + "attempt,replay_generation,dispatch_sequence,fencing_token,available_at,created_at," + "updated_at) VALUES (:id,:tenant,'inference_pipeline',:run,:logical,:payload,'ready'," + "0,0,1,0,:now,:now,:now)" + ), + { + "id": task_id, + "tenant": tenant, + "run": run_id, + "logical": f"inference_pipeline:{run_id}", + "payload": canonical_json(payload), + "now": now, + }, + ) + event = { + "schema_version": "1.1", + "tenant_id": tenant, + "trace_id": trace.replace("-", ""), + "task_id": task_id, + "task_type": "inference_pipeline", + "resource_id": run_id, + "replay_generation": 0, + "dispatch_sequence": 1, + "created_at": timestamp(now), + "payload": payload, + } + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) VALUES " + "(:id,:tenant,'TaskDispatch','task_run',:task,1,:payload,'pending',:now,:now,:now)" + ), + { + "id": str(uuid4()), + "tenant": tenant, + "task": task_id, + "payload": canonical_json(event), + "now": now, + }, + ) + return task_id + + +def _projection(connection, tenant, run_id): + row = ( + connection.execute( + text( + "SELECT r.id,r.created_at,r.updated_at,r.version,r.item_id,r.status,r.stage," + "r.submission_revision,r.model_bundle_id,r.dictionary_version_id,r.rule_bundle_id," + "r.pipeline_version,r.result_hash,r.error_code,t.id AS job_id " + "FROM inference_runs r JOIN task_runs t ON t.tenant_id=r.tenant_id " + "AND t.resource_id=r.id AND t.task_type='inference_pipeline' " + "WHERE r.tenant_id=:tenant AND r.id=:id" + ), + {"tenant": tenant, "id": run_id}, + ) + .mappings() + .first() + ) + if row is None: + raise not_found() + images = ( + connection.execute( + text( + "SELECT image_id FROM run_images WHERE tenant_id=:tenant AND run_id=:run " + "ORDER BY ordinal" + ), + {"tenant": tenant, "run": run_id}, + ) + .scalars() + .all() + ) + value = dict(row) + value["created_at"] = timestamp(value["created_at"]) + value["updated_at"] = timestamp(value["updated_at"]) + value["input_image_ids"] = list(images) + value["is_simulated"] = True + return value + + +def submit(connection, actor, item_id, expected_version, selections, request_id): + require_transaction(connection) + row = _item(connection, actor.tenant_id, item_id) + if row is None: + raise not_found() + item = Item( + id=row["id"], + tenant_id=row["tenant_id"], + laboratory_id=row["laboratory_id"], + version=row["version"], + status=row["status"], + inspection_status=row["inspection_status"], + creator_id=row["creator_id"], + location_id=row["location_id"], + current_run_id=row["current_run_id"], + current_evaluation_id=row["current_evaluation_id"], + ) + authorize(actor, Permission.CAPTURE, item.laboratory_id, creator_id=item.creator_id) + findings = _finding_rows(connection, actor.tenant_id, item_id, item.laboratory_id) + loaded, image_rows = _images(connection, actor.tenant_id, item_id, selections) + # The image schema binds images to the item and laboratory. Location is + # therefore the item's server-owned location, never a client field. + loaded = tuple( + Image( + id=image.id, + tenant_id=image.tenant_id, + laboratory_id=image.laboratory_id, + owner_type=image.owner_type, + owner_id=image.owner_id, + location_id=item.location_id, + status=image.status, + available=image.available, + ) + for image in loaded + ) + activation = _activation(connection, actor.tenant_id, item.laboratory_id) + submit_item( + actor, + item, + expected_version=expected_version, + images=loaded, + selections=tuple(selections), + current_findings=findings, + ) + now = connection.scalar(text("SELECT UTC_TIMESTAMP(3)")) + old_run = item.current_run_id + old_evaluation = item.current_evaluation_id + if old_run: + connection.execute( + text( + "UPDATE inference_runs SET status='superseded',version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND item_id=:item AND id=:run " + "AND status NOT IN ('superseded')" + ), + {"tenant": actor.tenant_id, "item": item.id, "run": old_run, "now": now}, + ) + if old_evaluation: + connection.execute( + text( + "UPDATE rule_evaluations SET status='superseded',version=version+1," + "updated_at=:now " + "WHERE tenant_id=:tenant AND item_id=:item AND id=:evaluation " + "AND status NOT IN ('superseded')" + ), + { + "tenant": actor.tenant_id, + "item": item.id, + "evaluation": old_evaluation, + "now": now, + }, + ) + connection.execute( + text( + "UPDATE findings SET superseded_at=:now,version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND item_id=:item AND superseded_at IS NULL" + ), + {"tenant": actor.tenant_id, "item": item.id, "now": now}, + ) + run_id, task_id = str(uuid4()), str(uuid4()) + revision = row["submission_revision"] + 1 + pinned_images = [ + { + "image_id": selection.image_id, + "role": selection.role, + "parent_image_id": selection.parent_image_id, + } + for selection in selections + ] + reference_date = now.date() + config = { + "model_bundle_id": activation["model_bundle_id"], + "dictionary_version_id": activation["dictionary_version_id"], + "rule_bundle_id": activation["rule_bundle_id"], + "pipeline_version": activation["pipeline_version"], + "device_profile": activation["device_profile"], + "reference_date": reference_date.isoformat(), + "images": pinned_images, + } + input_hash = hashlib.sha256(canonical_json(config).encode()).hexdigest() + connection.execute( + text( + "INSERT INTO inference_runs (id,tenant_id,item_id,laboratory_id,submission_revision," + "input_hash,model_bundle_id,dictionary_version_id,rule_bundle_id,pipeline_version," + "device_profile,reference_date,status,stage,created_at,updated_at) VALUES " + "(:id,:tenant,:item,:lab,:revision,:hash,:model,:dictionary,:rules,:pipeline,:device,:ref," + "'queued','queued',:now,:now)" + ), + { + "id": run_id, + "tenant": actor.tenant_id, + "item": item.id, + "lab": item.laboratory_id, + "revision": revision, + "hash": input_hash, + "model": activation["model_bundle_id"], + "dictionary": activation["dictionary_version_id"], + "rules": activation["rule_bundle_id"], + "pipeline": activation["pipeline_version"], + "device": activation["device_profile"], + "ref": reference_date, + "now": now, + }, + ) + for ordinal, (selection, image) in enumerate(zip(selections, image_rows)): + connection.execute( + text( + "INSERT INTO run_images (id,tenant_id,item_id,run_id,image_id,ordinal,role," + "parent_image_id,analysis_sha256) " + "VALUES (:id,:tenant,:item,:run,:image,:ordinal,:role,:parent,:sha)" + ), + { + "id": str(uuid4()), + "tenant": actor.tenant_id, + "item": item.id, + "run": run_id, + "image": image["id"], + "ordinal": ordinal, + "role": selection.role, + "parent": selection.parent_image_id, + "sha": image["analysis_sha256"], + }, + ) + payload = { + "run_id": run_id, + "submission_revision": revision, + "model_bundle_id": activation["model_bundle_id"], + "dictionary_version_id": activation["dictionary_version_id"], + "rule_bundle_id": activation["rule_bundle_id"], + } + connection.execute( + text( + "INSERT INTO task_runs (id,tenant_id,task_type,resource_id,logical_key,payload,state," + "attempt,replay_generation,dispatch_sequence,fencing_token,available_at,created_at," + "updated_at) VALUES (:id,:tenant,'inference_pipeline',:run,:logical,:payload,'ready'," + "0,0,1,0,:now,:now,:now)" + ), + { + "id": task_id, + "tenant": actor.tenant_id, + "run": run_id, + "logical": f"inference_pipeline:{run_id}", + "payload": canonical_json(payload), + "now": now, + }, + ) + event = { + "schema_version": "1.1", + "tenant_id": actor.tenant_id, + "trace_id": request_id.replace("-", ""), + "task_id": task_id, + "task_type": "inference_pipeline", + "resource_id": run_id, + "replay_generation": 0, + "dispatch_sequence": 1, + "created_at": timestamp(now), + "payload": payload, + } + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) " + "VALUES (:id,:tenant,'TaskDispatch','task_run',:task,1,:payload,'pending'," + ":now,:now,:now)" + ), + { + "id": str(uuid4()), + "tenant": actor.tenant_id, + "task": task_id, + "payload": canonical_json(event), + "now": now, + }, + ) + connection.execute( + text( + "UPDATE inspection_items SET status='queued',submission_revision=:revision," + "current_run_id=:run," + "current_fact_revision_id=NULL,current_evaluation_id=NULL,review_outcome=NULL,reviewed_by=NULL," + "reviewed_at=NULL,version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:item AND version=:version" + ), + { + "tenant": actor.tenant_id, + "item": item.id, + "revision": revision, + "run": run_id, + "now": now, + "version": item.version, + }, + ) + append_audit( + connection, + principal=actor, + action="inspection_item.submit", + resource_type="inspection_item", + resource_id=item.id, + changes={ + "status": "queued", + "run_id": run_id, + "submission_revision": revision, + }, + reason="Submit inspection item for inference", + request_id=request_id, + ) + return _projection(connection, actor.tenant_id, run_id) diff --git a/packages/persistence/job_execution.py b/packages/persistence/job_execution.py new file mode 100644 index 0000000..6f25ff4 --- /dev/null +++ b/packages/persistence/job_execution.py @@ -0,0 +1,390 @@ +"""validate_image execution fencing. Lock owner -> upload -> image -> task. + +All entry points require a caller-owned short transaction. Broker and object I/O +must remain outside it. Unsupported task types never acquire execution leases. +""" + +from datetime import timedelta +from types import SimpleNamespace +from uuid import uuid4 + +from sqlalchemy import text + +from packages.domain.image_validation import ValidatedImage, validate_result +from packages.domain.job_execution import ( + TECHNICAL_ERRORS, + ImageInput, + ImageLease, + InvalidDispatch, + LeaseLost, + retry_delay, +) +from packages.domain.security import ServiceError +from packages.domain.uploads import ( + CAPTURE_STATES, + EVIDENCE_STATES, + MAX_BYTES, + MIME_TYPES, + digest, + identifier, + owner_reference, + staging_key, +) +from packages.domain.workflow import Item +from packages.persistence.dispatch import db_now, decoded, validate_dispatch +from packages.persistence.transaction import require_transaction +from packages.persistence.uploads import UploadRepository + + +def _row(connection, table, tenant, row_id, *, lock=False): + # Identifiers are internal constants, never accepted from a message. + return ( + connection.execute( + text( + f"SELECT * FROM {table} WHERE tenant_id=:tenant AND id=:id" + + (" FOR UPDATE" if lock else "") + ), + {"tenant": tenant, "id": row_id}, + ) + .mappings() + .first() + ) + + +def _load(connection, tenant, task_id): + require_transaction(connection) + tenant_status = connection.scalar( + text("SELECT status FROM tenants WHERE id=:tenant FOR SHARE"), {"tenant": tenant} + ) + locator = _row(connection, "task_runs", tenant, task_id) + if locator is None or locator["task_type"] != "validate_image": + return None, None + source = _row(connection, "asset_images", tenant, locator["resource_id"]) + current = None + if source is not None: + owner_type, owner_id = owner_reference(source) + try: + owner = UploadRepository().owner( + connection, SimpleNamespace(tenant_id=tenant), owner_type, owner_id + ) + except ServiceError as error: + if error.status != 404: + raise + owner = None + upload = _row(connection, "uploads", tenant, source["upload_id"], lock=True) + image = _row(connection, "asset_images", tenant, source["id"], lock=True) + owner_valid = owner is not None and ( + ( + isinstance(owner, Item) + and owner.status in CAPTURE_STATES + and owner.inspection_status in {"draft", "in_progress"} + ) + or (not isinstance(owner, Item) and owner.status in EVIDENCE_STATES) + ) + if ( + tenant_status == "active" + and owner_valid + and upload is not None + and image is not None + and owner_reference(image) == (owner_type, owner_id) + and owner_reference(upload) == (owner_type, owner_id) + and image["upload_id"] == upload["id"] + and image["laboratory_id"] == upload["laboratory_id"] == owner.laboratory_id + and image["status"] == upload["status"] == "validating" + and image["original_key"] == upload["object_key"] == staging_key(tenant, upload["id"]) + and image["original_sha256"] == upload["expected_sha256"] + and image["mime_type"] == upload["mime_type"] in MIME_TYPES + and 1 <= upload["size_bytes"] <= MAX_BYTES + and image["original_object_version"] not in {"", "null"} + ): + digest(upload["expected_sha256"]) + current = ImageInput( + image["id"], + upload["id"], + image["laboratory_id"], + owner_type, + owner_id, + image["original_key"], + image["original_object_version"], + upload["expected_sha256"], + upload["size_bytes"], + upload["mime_type"], + image["version"], + upload["version"], + ) + task = _row(connection, "task_runs", tenant, task_id, lock=True) + if task is None or task["task_type"] != "validate_image": + return None, None + payload = decoded(task["payload"]) + if ( + current is None + or task["resource_id"] != current.image_id + or payload != {"image_id": current.image_id, "upload_id": current.upload_id} + or task["logical_key"] != f"validate_image:{current.image_id}" + ): + current = None + return task, current + + +def _attempt(connection, task, status, code, now): + count = connection.execute( + text( + "UPDATE task_attempts SET status=:status,error_code=:code,finished_at=:now," + "updated_at=:now,version=version+1 WHERE tenant_id=:tenant AND task_id=:id " + "AND replay_generation=:generation AND attempt=:attempt AND fencing_token=:token " + "AND lease_owner=:owner AND status='running'" + ), + { + "tenant": task["tenant_id"], + "id": task["id"], + "generation": task["replay_generation"], + "attempt": task["attempt"], + "token": task["fencing_token"], + "owner": task["lease_owner"], + "status": status, + "code": code, + "now": now, + }, + ).rowcount + if count != 1: + raise RuntimeError("Missing current task attempt") + + +def _end(connection, task, *, state, code, now, available=None, abandoned=False): + if task["state"] == "leased": + _attempt(connection, task, "abandoned" if abandoned else "failed", code, now) + connection.execute( + text( + "UPDATE task_runs SET state=:state,last_error_code=:code,lease_owner=NULL," + "lease_until=NULL,heartbeat_at=NULL,fencing_token=fencing_token+1," + "available_at=:available,finished_at=:finished,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "tenant": task["tenant_id"], + "id": task["id"], + "state": state, + "code": code, + "available": available or now, + "finished": None if state == "retry_wait" else now, + "now": now, + }, + ) + + +def _settle(connection, task, current, now, code, *, abandoned=False, delay=retry_delay): + if current is None: + state, code = "failed", "LEASE_LOST" + elif TECHNICAL_ERRORS[code]: + state = "retry_wait" if task["attempt"] < 4 else "dead_letter" + else: + state = "failed" + _end( + connection, + task, + state=state, + code=code, + now=now, + abandoned=abandoned, + available=now + delay(task["attempt"]) if state == "retry_wait" else None, + ) + return state + + +def claim(connection, message, owner): + validate_dispatch(message) + identifier(owner) + task, current = _load(connection, message["tenant_id"], message["task_id"]) + if task is None: + return None + # Generation dominates sequence: any prior generation is stale even if its + # sequence is numerically higher; a future generation is always invalid. + generation, sequence = message["replay_generation"], message["dispatch_sequence"] + if generation < task["replay_generation"]: + return None + if generation > task["replay_generation"]: + raise InvalidDispatch() + if sequence < task["dispatch_sequence"]: + return None + if sequence > task["dispatch_sequence"]: + raise InvalidDispatch() + if message["resource_id"] != task["resource_id"] or message["payload"] != decoded( + task["payload"] + ): + raise InvalidDispatch() + now = db_now(connection) + if task["state"] not in {"ready", "retry_wait"} or task["available_at"] > now: + return None + if current is None: + _end(connection, task, state="failed", code="LEASE_LOST", now=now) + return None + if task["attempt"] >= 4: + _end(connection, task, state="dead_letter", code="STAGE_TIMEOUT", now=now) + return None + attempt, token = task["attempt"] + 1, task["fencing_token"] + 1 + connection.execute( + text( + "UPDATE task_runs SET state='leased',attempt=:attempt,fencing_token=:token," + "lease_owner=:owner,lease_until=:until,heartbeat_at=:now," + "started_at=COALESCE(started_at,:now),finished_at=NULL,last_error_code=NULL," + "updated_at=:now,version=version+1 WHERE tenant_id=:tenant AND id=:id" + ), + { + "tenant": task["tenant_id"], + "id": task["id"], + "attempt": attempt, + "token": token, + "owner": owner, + "until": now + timedelta(seconds=60), + "now": now, + }, + ) + connection.execute( + text( + "INSERT INTO task_attempts (id,tenant_id,task_id,replay_generation,attempt," + "fencing_token,lease_owner,started_at,status,created_at,updated_at) VALUES " + "(:id,:tenant,:task,:generation,:attempt,:token,:owner,:now,'running',:now,:now)" + ), + { + "id": str(uuid4()), + "tenant": task["tenant_id"], + "task": task["id"], + "generation": task["replay_generation"], + "attempt": attempt, + "token": token, + "owner": owner, + "now": now, + }, + ) + return ImageLease( + task["tenant_id"], task["id"], owner, token, task["replay_generation"], attempt, current + ) + + +def _owned(task, lease, now): + return ( + task is not None + and task["state"] == "leased" + and task["lease_owner"] == lease.owner + and task["fencing_token"] == lease.token + and task["replay_generation"] == lease.generation + and task["attempt"] == lease.attempt + and task["lease_until"] is not None + and task["lease_until"] > now + ) + + +def heartbeat(connection, lease): + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now): + return False + if current != lease.input: + _settle(connection, task, None, now, "INTERNAL_ERROR") + return False + connection.execute( + text( + "UPDATE task_runs SET lease_until=:until,heartbeat_at=:now,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "tenant": lease.tenant_id, + "id": lease.task_id, + "until": now + timedelta(seconds=60), + "now": now, + }, + ) + return True + + +def fail(connection, lease, code, *, delay=retry_delay): + if code not in TECHNICAL_ERRORS: + raise ValueError("Use a validation outcome for content errors") + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now): + return False + _settle(connection, task, current if current == lease.input else None, now, code, delay=delay) + return True + + +def commit_result(connection, lease, write_result): + """Server-owned handler writes image/upload/owner/audit/event in this transaction. + + No public route or fixture supplies this callback. Raising rolls back all + handler writes and leaves the leased task for failure handling or recovery. + """ + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or current != lease.input: + raise LeaseLost() + write_result(connection, lease.input) + # A callback cannot manufacture succeeded without a terminal image result. + image = _row(connection, "asset_images", lease.tenant_id, current.image_id, lock=True) + upload = _row(connection, "uploads", lease.tenant_id, current.upload_id, lock=True) + if image["status"] not in {"ready", "rejected"} or upload["status"] != image["status"]: + raise RuntimeError("Handler did not persist a validation outcome") + if image["status"] == "ready": + if image["original_sha256"] != current.expected_sha256: + raise ValueError("Ready result changed original digest") + validate_result( + lease.tenant_id, + current, + ValidatedImage( + image["original_key"], + image["original_object_version"], + image["analysis_key"], + image["analysis_object_version"], + image["analysis_sha256"], + image["width"], + image["height"], + ), + ) + now = db_now(connection) + if not _owned( + _row(connection, "task_runs", lease.tenant_id, lease.task_id, lock=True), lease, now + ): + raise LeaseLost() + _attempt(connection, task, "succeeded", None, now) + connection.execute( + text( + "UPDATE task_runs SET state='succeeded',lease_owner=NULL,lease_until=NULL," + "heartbeat_at=NULL,finished_at=:now,last_error_code=NULL,updated_at=:now," + "version=version+1 WHERE tenant_id=:tenant AND id=:id" + ), + {"tenant": lease.tenant_id, "id": lease.task_id, "now": now}, + ) + + +def expired_candidates(connection, limit=100): + require_transaction(connection) + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("Batch limit must be 1..100") + # Discovery only. Never lock task before the domain aggregate. + return [ + dict(row) + for row in connection.execute( + text( + "SELECT tenant_id,id,fencing_token,replay_generation FROM task_runs " + "WHERE task_type='validate_image' AND state='leased' " + "AND lease_until<=UTC_TIMESTAMP(3) ORDER BY lease_until,id LIMIT :limit" + ), + {"limit": limit}, + ).mappings() + ] + + +def recover(connection, candidate, *, delay=retry_delay): + task, current = _load(connection, candidate["tenant_id"], candidate["id"]) + now = db_now(connection) + if ( + task is None + or task["state"] != "leased" + or task["lease_until"] is None + or task["lease_until"] > now + or task["fencing_token"] != candidate["fencing_token"] + or task["replay_generation"] != candidate["replay_generation"] + ): + return False + _settle(connection, task, current, now, "STAGE_TIMEOUT", abandoned=True, delay=delay) + return True diff --git a/packages/persistence/jobs.py b/packages/persistence/jobs.py new file mode 100644 index 0000000..137404e --- /dev/null +++ b/packages/persistence/jobs.py @@ -0,0 +1,151 @@ +"""Image-job read models and atomic generation replay, using domain-first locks.""" + +from uuid import UUID, uuid4 + +from sqlalchemy import text + +from packages.domain.security import Permission, authorize, not_found +from packages.persistence import job_execution +from packages.persistence.dispatch import db_now, decoded, validate_dispatch +from packages.persistence.foundations import BASE, project +from packages.persistence.security import append_audit +from packages.persistence.users import timestamp +from packages.shared.json_hash import canonical_json + +FIELDS = ( + BASE + ",task_type,resource_id,state,attempt,replay_generation,available_at,last_error_code" +) +COLUMNS = ",".join("job." + name for name in FIELDS.split(",")) +FROM = ( + "task_runs job JOIN asset_images image ON image.tenant_id=job.tenant_id " + "AND image.id=job.resource_id" +) + + +def job_projection(row): + result = project({name: row[name] for name in FIELDS.split(",")}) + result["available_at"] = timestamp(row["available_at"]) + return result + + +class JobRepository: + def get(self, connection, actor, job_id): + row = ( + connection.execute( + text( + f"SELECT {COLUMNS},image.laboratory_id FROM {FROM} WHERE job.tenant_id=:tenant " + "AND job.id=:id AND job.task_type='validate_image'" + ), + {"tenant": actor.tenant_id, "id": job_id}, + ) + .mappings() + .first() + ) + if row is None: + raise not_found() + authorize(actor, Permission.READ, row["laboratory_id"]) + return job_projection(row) + + def dead_letters(self, connection, actor, page, size, laboratory_id=None): + authorize(actor, Permission.ADMIN) + params = {"tenant": actor.tenant_id} + where = ( + "job.tenant_id=:tenant AND job.task_type='validate_image' " + "AND job.state IN ('failed','dead_letter')" + ) + if laboratory_id is not None: + # Plain snapshot read; do not change RR paging into a locking query. + lab = connection.scalar( + text("SELECT id FROM laboratories WHERE tenant_id=:tenant AND id=:id"), + {"tenant": actor.tenant_id, "id": laboratory_id}, + ) + if lab is None: + raise not_found() + params["lab"] = laboratory_id + where += " AND image.laboratory_id=:lab" + total = connection.scalar(text(f"SELECT COUNT(*) FROM {FROM} WHERE {where}"), params) + rows = connection.execute( + text( + f"SELECT {COLUMNS} FROM {FROM} WHERE {where} " + "ORDER BY job.created_at DESC,job.id DESC " + "LIMIT :limit OFFSET :offset" + ), + {**params, "limit": size, "offset": (page - 1) * size}, + ).mappings() + return { + "items": [job_projection(row) for row in rows], + "total": total, + "page": page, + "page_size": size, + } + + def replay_source(self, connection, actor, job_id): + # Same tenant -> owner -> upload -> image -> task order as claim/commit/recover. + return job_execution._load(connection, actor.tenant_id, job_id) + + def replay(self, connection, actor, task, reason, request_id): + now = db_now(connection) + generation, sequence = task["replay_generation"] + 1, task["dispatch_sequence"] + 1 + version = task["version"] + 1 + message = validate_dispatch( + { + "schema_version": "1.1", + "tenant_id": actor.tenant_id, + "trace_id": UUID(request_id).hex, + "task_id": task["id"], + "task_type": "validate_image", + "resource_id": task["resource_id"], + "replay_generation": generation, + "dispatch_sequence": sequence, + "created_at": timestamp(now), + "payload": decoded(task["payload"]), + } + ) + connection.execute( + text( + "UPDATE task_runs SET state='ready',attempt=0,replay_generation=:generation," + "fencing_token=fencing_token+1,dispatch_sequence=:sequence,lease_owner=NULL,lease_until=NULL," + "heartbeat_at=NULL,started_at=NULL,finished_at=NULL,last_error_code=NULL,available_at=:now," + "last_dispatched_at=:now,updated_at=:now,version=:version " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "tenant": actor.tenant_id, + "id": task["id"], + "generation": generation, + "sequence": sequence, + "version": version, + "now": now, + }, + ) + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) VALUES " + "(:event,:tenant,'TaskDispatch','task_run',:task,:version,:payload,'pending',:now,:now,:now)" + ), + { + "event": str(uuid4()), + "tenant": actor.tenant_id, + "task": task["id"], + "version": version, + "payload": canonical_json(message), + "now": now, + }, + ) + append_audit( + connection, + principal=actor, + action="job.replay", + resource_type="job", + resource_id=task["id"], + changes={ + "state": "ready", + "previous_state": task["state"], + "replay_generation": generation, + "attempt": 0, + }, + reason=reason, + request_id=request_id, + ) + return self.get(connection, actor, task["id"]) diff --git a/packages/persistence/review.py b/packages/persistence/review.py new file mode 100644 index 0000000..632eba1 --- /dev/null +++ b/packages/persistence/review.py @@ -0,0 +1,917 @@ +"""Facts, findings, and review-decision persistence. + +Commands lock the inspection item first, then its current run/evaluation and +the finding rows. Every current-pointer change is fenced by the item's +expected version and emits an audit record in the same transaction. +""" + +import json +from datetime import date +from uuid import uuid4 + +from sqlalchemy import text + +from packages.domain.security import Permission, ServiceError, authorize, not_found, require_version +from packages.domain.workflow import ( + Finding, + cannot_determine_finding, + confirm_finding, + reject_finding, +) +from packages.persistence.dispatch import db_now +from packages.persistence.foundations import lab_filter, page_query, project +from packages.persistence.security import append_audit +from packages.persistence.transaction import require_transaction +from packages.persistence.users import timestamp +from packages.shared.json_hash import canonical_json + + +def _decoded(value): + if isinstance(value, str): + try: + return json.loads(value) + except (TypeError, ValueError): + return None + return value + + +def _domain_event( + connection, *, tenant, event_type, aggregate_type, aggregate_id, version, payload, now +): + event_id = str(uuid4()) + event = { + "schema_version": "1.1", + "tenant_id": tenant, + "trace_id": uuid4().hex, + "event_id": event_id, + "event_type": event_type, + "aggregate_type": aggregate_type, + "aggregate_id": aggregate_id, + "occurred_at": timestamp(now), + "aggregate_version": version, + "payload": payload, + } + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) VALUES " + "(:id,:tenant,:event_type,:aggregate_type,:aggregate_id,:version,:payload,'pending'," + ":now,:now,:now)" + ), + { + "id": event_id, + "tenant": tenant, + "event_type": event_type, + "aggregate_type": aggregate_type, + "aggregate_id": aggregate_id, + "version": version, + "payload": canonical_json(event), + "now": now, + }, + ) + return event_id + + +def _fact_projection(row): + value = project(dict(row)) + for field in ("entities", "relations", "dates"): + decoded = _decoded(value.get(field)) + value[field] = _public_facts(field, decoded if isinstance(decoded, list) else []) + value.pop("tenant_id", None) + value.pop("created_by", None) + return value + + +def _public_facts(field, values): + """Project the inference protocol's model observations to FactRevision facts.""" + result = [] + for item in values: + if not isinstance(item, dict): + continue + if field == "entities": + if {"entity_id", "source", "evidence"} <= set(item): + result.append(item) + continue + candidates = item.get("candidates") + candidates = candidates if isinstance(candidates, list) else [] + entity_id = ( + candidates[0].get("entity_id") + if item.get("resolution") == "resolved" + and len(candidates) == 1 + and isinstance(candidates[0], dict) + else None + ) + result.append( + { + "detection_id": item.get("detection_id"), + "entity_id": entity_id, + "resolution": item.get("resolution", "unknown"), + "source": "model", + "evidence": [ + { + "image_id": item.get("image_id"), + "detection_id": item.get("detection_id"), + "crop_id": None, + } + ], + } + ) + elif field == "relations": + if {"same_location", "source", "evidence"} <= set(item): + result.append(item) + continue + result.append( + { + "source_detection_id": item.get("source_detection_id"), + "target_detection_id": item.get("target_detection_id"), + "relation": item.get("relation", "unknown"), + "same_location": None, + "source": "model", + "evidence": [ + { + "image_id": item.get("image_id"), + "detection_id": item.get("source_detection_id"), + "crop_id": None, + } + ], + } + ) + elif field == "dates": + if {"kind", "source", "evidence"} <= set(item): + result.append(item) + continue + result.append( + { + "detection_id": item.get("detection_id"), + "kind": item.get("kind", "unknown"), + "value": item.get("value"), + "source": "model", + "evidence": [ + { + "image_id": item.get("image_id"), + "detection_id": item.get("detection_id"), + "crop_id": None, + } + ], + } + ) + return result + + +def _item(connection, tenant, item_id, *, lock=False): + return ( + connection.execute( + text( + "SELECT i.id,i.tenant_id,i.laboratory_id,i.version,i.status," + "i.current_run_id,i.current_fact_revision_id,i.current_evaluation_id," + "ins.status AS inspection_status,ins.inspector_id AS creator_id " + "FROM inspection_items i JOIN inspections ins ON ins.tenant_id=i.tenant_id " + "AND ins.id=i.inspection_id AND ins.laboratory_id=i.laboratory_id " + "WHERE i.tenant_id=:tenant AND i.id=:item" + (" FOR UPDATE" if lock else "") + ), + {"tenant": tenant, "item": item_id}, + ) + .mappings() + .first() + ) + + +def _fact(connection, tenant, fact_id, *, lock=False): + return ( + connection.execute( + text( + "SELECT id,created_at,updated_at,version,tenant_id,item_id,run_id,revision," + "entities,relations,dates,reason,created_by FROM fact_revisions " + "WHERE tenant_id=:tenant AND id=:fact" + (" FOR UPDATE" if lock else "") + ), + {"tenant": tenant, "fact": fact_id}, + ) + .mappings() + .first() + ) + + +def current_facts(connection, actor, item_id): + row = _item(connection, actor.tenant_id, item_id) + if row is None: + raise not_found() + authorize(actor, Permission.READ, row["laboratory_id"]) + if row["current_fact_revision_id"] is None: + raise not_found() + fact = _fact(connection, actor.tenant_id, row["current_fact_revision_id"]) + if fact is None or fact["item_id"] != item_id or fact["run_id"] != row["current_run_id"]: + raise not_found() + return _fact_projection(fact) + + +def authorize_item(connection, actor, item_id): + row = _item(connection, actor.tenant_id, item_id) + if row is None: + raise not_found() + authorize(actor, Permission.REVIEW, row["laboratory_id"]) + return row + + +def _task_projection(connection, tenant, task_id): + row = ( + connection.execute( + text( + "SELECT id,created_at,updated_at,version,task_type,resource_id,state,attempt," + "replay_generation,available_at,last_error_code FROM task_runs " + "WHERE tenant_id=:tenant AND id=:id" + ), + {"tenant": tenant, "id": task_id}, + ) + .mappings() + .first() + ) + if row is None: + raise not_found() + value = project(row) + value["available_at"] = timestamp(row["available_at"]) + return value + + +def _finding_evidence(raw, connection, tenant, run_id, *, ready_only=False): + refs = [] + readiness = ( + "AND ai.status='ready' AND ai.analysis_key IS NOT NULL " + "AND ai.analysis_object_version IS NOT NULL " + if ready_only + else "" + ) + run_image_ids = list( + connection.execute( + text( + "SELECT ri.image_id FROM run_images ri " + "JOIN asset_images ai ON ai.tenant_id=ri.tenant_id AND ai.id=ri.image_id " + "WHERE ri.tenant_id=:tenant AND ri.run_id=:run " + + readiness + + "ORDER BY ri.ordinal,ri.id" + ), + {"tenant": tenant, "run": run_id}, + ).scalars() + ) + run_images = set(run_image_ids) + raw = raw if isinstance(raw, dict) else {} + raw_facts = raw.get("raw_facts") if isinstance(raw.get("raw_facts"), dict) else {} + for source in (raw_facts.get("entities"), raw_facts.get("relations")): + if not isinstance(source, list): + continue + for item in source: + if not isinstance(item, dict): + continue + item_refs = item.get("evidence") if isinstance(item.get("evidence"), list) else [item] + for ref in item_refs: + if ( + not isinstance(ref, dict) + or ref.get("image_id") is None + or ref["image_id"] not in run_images + ): + continue + refs.append( + { + "image_id": ref["image_id"], + "detection_id": ref.get("detection_id") + or item.get("detection_id") + or item.get("source_detection_id"), + "crop_id": ref.get("crop_id"), + } + ) + if not refs: + image_id = run_image_ids[0] if run_image_ids else None + if image_id is not None: + refs.append({"image_id": image_id, "detection_id": None, "crop_id": None}) + unique = [] + seen = set() + for ref in refs: + key = (ref["image_id"], ref["detection_id"], ref["crop_id"]) + if key not in seen: + unique.append(ref) + seen.add(key) + if not unique: + raise ServiceError("STATE_CONFLICT", 409, "Finding evidence is unavailable") + return unique[:20] + + +def _finding_projection(connection, actor, row): + value = project(dict(row)) + raw = _decoded(row.get("evidence")) + value["evidence"] = _finding_evidence(raw, connection, actor.tenant_id, row["run_id"]) + value["task_id"] = row.get("task_id") + value["allowed_actions"] = [] + try: + authorize(actor, Permission.REVIEW, row["laboratory_id"]) + if row["superseded_at"] is None and row["status"] == "needs_review": + value["allowed_actions"] = [ + "confirmFinding", + "rejectFinding", + "cannotdetermineFinding", + ] + except ServiceError: + pass + value.pop("tenant_id", None) + value.pop("laboratory_id", None) + value["superseded_at"] = timestamp(row["superseded_at"]) if row["superseded_at"] else None + return value + + +_FINDING_COLUMNS = ( + "f.id,f.created_at,f.updated_at,f.version,f.tenant_id,f.item_id,f.run_id," + "f.fact_revision_id,f.rule_evaluation_id,f.rule_id,f.type,f.severity,f.status," + "f.explanation,f.evidence,f.superseded_at,f.confirmed_by,r.laboratory_id," + "rt.id AS task_id" +) + + +def _finding_row(connection, tenant, finding_id, *, lock=False): + return ( + connection.execute( + text( + f"SELECT {_FINDING_COLUMNS} FROM findings f " + "JOIN inference_runs r ON r.tenant_id=f.tenant_id AND r.item_id=f.item_id " + "AND r.id=f.run_id LEFT JOIN remediation_tasks rt ON rt.tenant_id=f.tenant_id " + "AND rt.finding_id=f.id WHERE f.tenant_id=:tenant AND f.id=:id" + + (" FOR UPDATE" if lock else "") + ), + {"tenant": tenant, "id": finding_id}, + ) + .mappings() + .first() + ) + + +def get_finding(connection, actor, finding_id): + row = _finding_row(connection, actor.tenant_id, finding_id) + if row is None: + raise not_found() + authorize(actor, Permission.READ, row["laboratory_id"]) + return _finding_projection(connection, actor, row) + + +def authorize_finding(connection, actor, finding_id): + row = _finding_row(connection, actor.tenant_id, finding_id) + if row is None: + raise not_found() + authorize(actor, Permission.REVIEW, row["laboratory_id"]) + return row + + +def list_findings( + connection, + actor, + page, + page_size, + laboratory_id=None, + current_only=True, + status=None, + severity=None, +): + params = {"tenant": actor.tenant_id} + where = "f.tenant_id=:tenant" + lab_filter(actor, "r.laboratory_id", params, laboratory_id) + if current_only: + where += " AND f.superseded_at IS NULL" + if status is not None: + where += " AND f.status=:status" + params["status"] = status + if severity is not None: + where += " AND f.severity=:severity" + params["severity"] = severity + return page_query( + connection, + "findings f JOIN inference_runs r ON r.tenant_id=f.tenant_id AND r.item_id=f.item_id " + "AND r.id=f.run_id LEFT JOIN remediation_tasks rt ON rt.tenant_id=f.tenant_id " + "AND rt.finding_id=f.id", + _FINDING_COLUMNS, + where, + params, + page, + page_size, + project_row=lambda row: _finding_projection(connection, actor, row), + order_by="f.created_at DESC,f.id DESC", + ) + + +def _finding_domain(row): + return Finding( + id=row["id"], + tenant_id=row["tenant_id"], + laboratory_id=row["laboratory_id"], + version=row["version"], + status=row["status"], + item_id=row["item_id"], + run_id=row["run_id"], + evaluation_id=row["rule_evaluation_id"], + superseded=row["superseded_at"] is not None, + evidence_ready=bool(_decoded(row.get("evidence"))), + ) + + +def _validate_fact_snapshot(connection, tenant, item_id, run, body): + """Validate a complete human fact replacement against the pinned run input.""" + arrays = { + "entities": body.get("entities"), + "relations": body.get("relations"), + "dates": body.get("dates"), + } + if any(not isinstance(value, list) for value in arrays.values()): + raise ServiceError("VALIDATION_ERROR", 422, "Facts must be complete arrays") + + result = _decoded(run.get("result")) + result = result if isinstance(result, dict) else {} + detections = result.get("detections") if isinstance(result.get("detections"), list) else [] + detection_ids = { + value.get("detection_id") + for value in detections + if isinstance(value, dict) and isinstance(value.get("detection_id"), str) + } + image_ids = { + value + for value in connection.execute( + text("SELECT image_id FROM run_images WHERE tenant_id=:tenant AND run_id=:run"), + {"tenant": tenant, "run": run["id"]}, + ).scalars() + } + crop_rows = ( + connection.execute( + text( + "SELECT image_id,crop_id,detection_id FROM image_derivatives " + "WHERE tenant_id=:tenant AND run_id=:run" + ), + {"tenant": tenant, "run": run["id"]}, + ) + .mappings() + .all() + ) + crops = {(row["image_id"], row["crop_id"], row["detection_id"]) for row in crop_rows} + crop_ids = {row["crop_id"] for row in crop_rows} + + def evidence_refs(value): + refs = value.get("evidence") if isinstance(value, dict) else None + if not isinstance(refs, list) or not refs: + raise ServiceError("VALIDATION_ERROR", 422, "Each fact needs evidence") + seen = set() + for ref in refs: + if not isinstance(ref, dict) or set(ref) != {"image_id", "detection_id", "crop_id"}: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid fact evidence") + image_id = ref["image_id"] + detection_id = ref["detection_id"] + crop_id = ref["crop_id"] + if image_id not in image_ids: + raise ServiceError( + "STATE_CONFLICT", 409, "Evidence image is not in the current run" + ) + if detection_id is not None and detection_id not in detection_ids: + raise ServiceError( + "STATE_CONFLICT", 409, "Evidence detection is not in the current run" + ) + if crop_id is not None: + if crop_id not in crop_ids or not any( + row[0] == image_id and row[1] == crop_id and row[2] == detection_id + for row in crops + ): + raise ServiceError( + "STATE_CONFLICT", 409, "Evidence crop is not in the current run" + ) + key = (image_id, detection_id, crop_id) + if key in seen: + raise ServiceError("VALIDATION_ERROR", 422, "Duplicate fact evidence") + seen.add(key) + + entity_keys = set() + for value in arrays["entities"]: + if not isinstance(value, dict) or set(value) != { + "detection_id", + "entity_id", + "resolution", + "source", + "evidence", + }: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid entity fact") + if value["detection_id"] not in detection_ids: + raise ServiceError("STATE_CONFLICT", 409, "Entity detection is not in the current run") + if value["source"] != "human": + raise ServiceError("VALIDATION_ERROR", 422, "Fact edits must use human source") + if value["resolution"] == "resolved" and value["entity_id"] is None: + raise ServiceError("VALIDATION_ERROR", 422, "Resolved entity needs entity_id") + if value["entity_id"] is not None: + known_entity = connection.scalar( + text( + "SELECT id FROM chemical_entities WHERE tenant_id=:tenant " + "AND dictionary_version_id=:dictionary AND id=:entity" + ), + { + "tenant": tenant, + "dictionary": run["dictionary_version_id"], + "entity": value["entity_id"], + }, + ) + if known_entity is None: + raise ServiceError("STATE_CONFLICT", 409, "Entity is outside the pinned dictionary") + key = value["detection_id"] + if key in entity_keys: + raise ServiceError("VALIDATION_ERROR", 422, "Duplicate entity fact") + entity_keys.add(key) + evidence_refs(value) + + relation_keys = set() + for value in arrays["relations"]: + if not isinstance(value, dict) or set(value) != { + "source_detection_id", + "target_detection_id", + "relation", + "same_location", + "source", + "evidence", + }: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid relation fact") + if ( + value["source_detection_id"] not in detection_ids + or value["target_detection_id"] not in detection_ids + ): + raise ServiceError( + "STATE_CONFLICT", 409, "Relation detection is not in the current run" + ) + if value["source"] != "human": + raise ServiceError("VALIDATION_ERROR", 422, "Fact edits must use human source") + if value["source_detection_id"] == value["target_detection_id"]: + raise ServiceError("VALIDATION_ERROR", 422, "Relation endpoints must differ") + if value["same_location"] is not True and value["relation"] != "unknown": + raise ServiceError( + "VALIDATION_ERROR", 422, "Known adjacency requires same_location=true" + ) + key = (value["source_detection_id"], value["target_detection_id"]) + if key in relation_keys: + raise ServiceError("VALIDATION_ERROR", 422, "Duplicate relation fact") + relation_keys.add(key) + evidence_refs(value) + + date_keys = set() + for value in arrays["dates"]: + if not isinstance(value, dict) or set(value) != { + "detection_id", + "kind", + "value", + "source", + "evidence", + }: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid date fact") + if value["detection_id"] not in detection_ids: + raise ServiceError("STATE_CONFLICT", 409, "Date detection is not in the current run") + if value["source"] != "human": + raise ServiceError("VALIDATION_ERROR", 422, "Fact edits must use human source") + if value["value"] is not None: + try: + date.fromisoformat(value["value"]) + except (TypeError, ValueError): + raise ServiceError("VALIDATION_ERROR", 422, "Invalid fact date") from None + key = (value["detection_id"], value["kind"]) + if key in date_keys: + raise ServiceError("VALIDATION_ERROR", 422, "Duplicate date fact") + date_keys.add(key) + evidence_refs(value) + + +def decide_finding(connection, actor, finding_id, body, operation, request_id): + row = _finding_row(connection, actor.tenant_id, finding_id, lock=True) + if row is None: + raise not_found() + authorize(actor, Permission.REVIEW, row["laboratory_id"]) + finding = _finding_domain(row) + evidence = _finding_evidence( + _decoded(row.get("evidence")), + connection, + actor.tenant_id, + row["run_id"], + ready_only=True, + ) + finding = Finding( + id=finding.id, + tenant_id=finding.tenant_id, + laboratory_id=finding.laboratory_id, + version=finding.version, + status=finding.status, + item_id=finding.item_id, + run_id=finding.run_id, + evaluation_id=finding.evaluation_id, + superseded=finding.superseded, + evidence_ready=bool(evidence), + ) + expected = body.get("expected_version") + reason = body.get("reason") + if operation == "confirm": + new_status = confirm_finding(actor, finding, expected_version=expected, reason=reason) + elif operation == "reject": + new_status = reject_finding(actor, finding, expected_version=expected, reason=reason) + elif operation == "cannot-determine": + new_status = cannot_determine_finding( + actor, + finding, + expected_version=expected, + reason=reason, + reason_code=body.get("reason_code"), + ) + else: + raise ServiceError("VALIDATION_ERROR", 422, "Unknown finding decision") + now = db_now(connection) + updated = connection.execute( + text( + "UPDATE findings SET status=:status,confirmed_by=:confirmed,version=version+1," + "updated_at=:now WHERE tenant_id=:tenant AND id=:id AND version=:version " + "AND superseded_at IS NULL" + ), + { + "status": str(new_status), + "confirmed": actor.user_id if str(new_status) == "confirmed" else None, + "now": now, + "tenant": actor.tenant_id, + "id": finding_id, + "version": expected, + }, + ).rowcount + if updated != 1: + raise ServiceError("VERSION_CONFLICT", 409, "Refresh the finding before retrying") + event_type = { + "confirmed": "FindingConfirmed", + "rejected": "FindingRejected", + }.get(str(new_status)) + if event_type is not None: + _domain_event( + connection, + tenant=actor.tenant_id, + event_type=event_type, + aggregate_type="finding", + aggregate_id=finding_id, + version=expected + 1, + payload={"finding_id": finding_id, "actor_id": actor.user_id}, + now=now, + ) + connection.execute( + text( + "INSERT INTO review_actions (id,tenant_id,actor_id,resource_type,resource_id,action," + "reason,before_version,after_version) VALUES (:id,:tenant,:actor,'finding',:resource," + ":action,:reason,:before,:after)" + ), + { + "id": str(uuid4()), + "tenant": actor.tenant_id, + "actor": actor.user_id, + "resource": finding_id, + "action": f"finding.{operation}", + "reason": reason, + "before": expected, + "after": expected + 1, + }, + ) + append_audit( + connection, + principal=actor, + action=f"finding.{operation}", + resource_type="finding", + resource_id=finding_id, + changes={ + "status": str(new_status), + "version": expected + 1, + **({"reason_code": body.get("reason_code")} if operation == "cannot-determine" else {}), + }, + reason=reason, + request_id=request_id, + ) + return _finding_projection( + connection, actor, _finding_row(connection, actor.tenant_id, finding_id) + ) + + +def edit_facts(connection, actor, item_id, body, request_id): + require_transaction(connection) + row = _item(connection, actor.tenant_id, item_id, lock=True) + if row is None: + raise not_found() + authorize(actor, Permission.REVIEW, row["laboratory_id"]) + require_version(row["version"], body.get("expected_version")) + if row["current_run_id"] is None or row["current_evaluation_id"] is None: + raise not_found() + if row["status"] != "needs_review": + raise ServiceError("STATE_CONFLICT", 409, "Facts can only be edited during review") + run = ( + connection.execute( + text( + "SELECT * FROM inference_runs WHERE tenant_id=:tenant AND id=:run " + "AND item_id=:item FOR UPDATE" + ), + { + "tenant": actor.tenant_id, + "run": row["current_run_id"], + "item": item_id, + }, + ) + .mappings() + .first() + ) + current_findings = ( + connection.execute( + text( + "SELECT id,status FROM findings WHERE tenant_id=:tenant AND item_id=:item " + "AND run_id=:run AND rule_evaluation_id=:evaluation AND superseded_at IS NULL " + "ORDER BY id FOR UPDATE" + ), + { + "tenant": actor.tenant_id, + "item": item_id, + "run": row["current_run_id"], + "evaluation": row["current_evaluation_id"], + }, + ) + .mappings() + .all() + ) + if any(value["status"] in {"confirmed", "dispatched", "closed"} for value in current_findings): + raise ServiceError("STATE_CONFLICT", 409, "Confirmed findings block fact revision") + evaluation = ( + connection.execute( + text( + "SELECT * FROM rule_evaluations WHERE tenant_id=:tenant AND id=:evaluation " + "AND item_id=:item AND run_id=:run FOR UPDATE" + ), + { + "tenant": actor.tenant_id, + "evaluation": row["current_evaluation_id"], + "item": item_id, + "run": row["current_run_id"], + }, + ) + .mappings() + .first() + ) + if ( + run is None + or run["status"] != "needs_review" + or run["stage"] != "done" + or evaluation is None + or evaluation["status"] != "completed" + ): + raise ServiceError("STATE_CONFLICT", 409, "Current evaluation is not complete") + _validate_fact_snapshot(connection, actor.tenant_id, item_id, run, body) + now = db_now(connection) + revision = connection.scalar( + text( + "SELECT COALESCE(MAX(revision),0)+1 FROM fact_revisions " + "WHERE tenant_id=:tenant AND run_id=:run" + ), + {"tenant": actor.tenant_id, "run": run["id"]}, + ) + fact_id = str(uuid4()) + connection.execute( + text( + "INSERT INTO fact_revisions (id,tenant_id,item_id,run_id,revision,entities,relations," + "dates,reason,created_by) VALUES (:id,:tenant,:item,:run,:revision," + ":entities,:relations,:dates,:reason,:actor)" + ), + { + "id": fact_id, + "tenant": actor.tenant_id, + "item": item_id, + "run": run["id"], + "revision": revision, + "entities": canonical_json(body["entities"]), + "relations": canonical_json(body["relations"]), + "dates": canonical_json(body["dates"]), + "reason": body["reason"], + "actor": actor.user_id, + }, + ) + connection.execute( + text( + "UPDATE rule_evaluations SET status='superseded',version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:evaluation" + ), + {"tenant": actor.tenant_id, "evaluation": evaluation["id"], "now": now}, + ) + connection.execute( + text( + "UPDATE findings SET superseded_at=:now,version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND item_id=:item AND superseded_at IS NULL" + ), + {"tenant": actor.tenant_id, "item": item_id, "now": now}, + ) + new_evaluation = str(uuid4()) + connection.execute( + text( + "INSERT INTO rule_evaluations (id,tenant_id,item_id,run_id,fact_revision_id," + "rule_bundle_id,reference_date,status) VALUES (:id,:tenant,:item,:run,:fact,:bundle," + ":reference,'queued')" + ), + { + "id": new_evaluation, + "tenant": actor.tenant_id, + "item": item_id, + "run": run["id"], + "fact": fact_id, + "bundle": run["rule_bundle_id"], + "reference": run["reference_date"], + }, + ) + task_id = str(uuid4()) + payload = { + "run_id": run["id"], + "fact_revision_id": fact_id, + "rule_bundle_id": run["rule_bundle_id"], + } + connection.execute( + text( + "INSERT INTO task_runs (id,tenant_id,task_type,resource_id,logical_key,payload,state," + "attempt,replay_generation,dispatch_sequence,fencing_token,available_at,created_at," + "updated_at) VALUES (:id,:tenant,'rule_evaluation',:run,:logical,:payload,'ready'," + "0,0,1,0,:now,:now,:now)" + ), + { + "id": task_id, + "tenant": actor.tenant_id, + "run": run["id"], + "logical": f"rule_evaluation:{new_evaluation}", + "payload": canonical_json(payload), + "now": now, + }, + ) + event = { + "schema_version": "1.1", + "tenant_id": actor.tenant_id, + "trace_id": uuid4().hex, + "task_id": task_id, + "task_type": "rule_evaluation", + "resource_id": run["id"], + "replay_generation": 0, + "dispatch_sequence": 1, + "created_at": timestamp(now), + "payload": payload, + } + connection.execute( + text( + "INSERT INTO outbox_events (id,tenant_id,event_type,aggregate_type,aggregate_id," + "aggregate_version,payload,state,available_at,created_at,updated_at) VALUES " + "(:id,:tenant,'TaskDispatch','task_run',:task,1,:payload,'pending',:now,:now,:now)" + ), + { + "id": str(uuid4()), + "tenant": actor.tenant_id, + "task": task_id, + "payload": canonical_json(event), + "now": now, + }, + ) + updated = connection.execute( + text( + "UPDATE inference_runs SET status='processing',stage='rules',finished_at=NULL," + "updated_at=:now,version=version+1 WHERE tenant_id=:tenant AND id=:run" + ), + {"tenant": actor.tenant_id, "run": run["id"], "now": now}, + ).rowcount + if updated != 1: + raise ServiceError("STATE_CONFLICT", 409, "Current inference run changed during facts edit") + updated = connection.execute( + text( + "UPDATE inspection_items SET status='processing',current_fact_revision_id=:fact," + "current_evaluation_id=:evaluation,review_outcome=NULL,reviewed_by=NULL," + "reviewed_at=NULL,version=version+1,updated_at=:now WHERE tenant_id=:tenant " + "AND id=:item AND version=:version" + ), + { + "tenant": actor.tenant_id, + "item": item_id, + "fact": fact_id, + "evaluation": new_evaluation, + "now": now, + "version": body.get("expected_version"), + }, + ).rowcount + if updated != 1: + raise ServiceError("VERSION_CONFLICT", 409, "Refresh the item before retrying") + _domain_event( + connection, + tenant=actor.tenant_id, + event_type="FactsRevised", + aggregate_type="inspection_item", + aggregate_id=item_id, + version=body.get("expected_version") + 1, + payload={ + "run_id": run["id"], + "fact_revision_id": fact_id, + "rule_bundle_id": run["rule_bundle_id"], + }, + now=now, + ) + append_audit( + connection, + principal=actor, + action="inspection_item.facts_edit", + resource_type="inspection_item", + resource_id=item_id, + changes={ + "fact_revision_id": fact_id, + "evaluation_id": new_evaluation, + "status": "processing", + }, + reason=body["reason"], + request_id=request_id, + ) + return _task_projection(connection, actor.tenant_id, task_id) diff --git a/packages/persistence/rule_execution.py b/packages/persistence/rule_execution.py new file mode 100644 index 0000000..a7761a3 --- /dev/null +++ b/packages/persistence/rule_execution.py @@ -0,0 +1,599 @@ +"""Fenced persistence for rule_evaluation tasks.""" + +import json +from datetime import timedelta +from uuid import uuid4 + +from sqlalchemy import text + +from packages.domain.job_execution import retry_delay +from packages.domain.rule_execution import RULE_RETRYABLE, RuleInput, RuleLease, RuleLeaseLost +from packages.persistence.dispatch import db_now, decoded, validate_dispatch +from packages.persistence.transaction import require_transaction +from packages.rules.evaluator import RuleDefinitionError, evaluate_bundle, normalize_facts +from packages.shared.json_hash import canonical_json, sha256_json + + +def _task(connection, tenant, task_id, *, lock=False): + return ( + connection.execute( + text( + "SELECT * FROM task_runs WHERE tenant_id=:tenant AND id=:id" + + (" FOR UPDATE" if lock else "") + ), + {"tenant": tenant, "id": task_id}, + ) + .mappings() + .first() + ) + + +def _load(connection, tenant, task_id): + require_transaction(connection) + task = _task(connection, tenant, task_id) + if task is None or task["task_type"] != "rule_evaluation": + return None, None + run = ( + connection.execute( + text("SELECT * FROM inference_runs WHERE tenant_id=:tenant AND id=:run"), + {"tenant": tenant, "run": task["resource_id"]}, + ) + .mappings() + .first() + ) + if run is None: + return None, None + item = ( + connection.execute( + text("SELECT * FROM inspection_items WHERE tenant_id=:tenant AND id=:item FOR UPDATE"), + {"tenant": tenant, "item": run["item_id"]}, + ) + .mappings() + .first() + ) + if item is not None: + run = ( + connection.execute( + text("SELECT * FROM inference_runs WHERE tenant_id=:tenant AND id=:run FOR UPDATE"), + {"tenant": tenant, "run": run["id"]}, + ) + .mappings() + .first() + ) + fact = None + evaluation = None + if item is not None and item["current_fact_revision_id"]: + fact = ( + connection.execute( + text( + "SELECT * FROM fact_revisions WHERE tenant_id=:tenant AND id=:fact " + "AND item_id=:item AND run_id=:run FOR UPDATE" + ), + { + "tenant": tenant, + "fact": item["current_fact_revision_id"], + "item": run["item_id"], + "run": run["id"], + }, + ) + .mappings() + .first() + ) + if item is not None and item["current_evaluation_id"]: + evaluation = ( + connection.execute( + text( + "SELECT * FROM rule_evaluations WHERE tenant_id=:tenant AND id=:evaluation " + "AND item_id=:item AND run_id=:run FOR UPDATE" + ), + { + "tenant": tenant, + "evaluation": item["current_evaluation_id"], + "item": run["item_id"], + "run": run["id"], + }, + ) + .mappings() + .first() + ) + task = _task(connection, tenant, task_id, lock=True) + if task is None: + return None, None + payload = decoded(task["payload"]) + if ( + item is None + or fact is None + or evaluation is None + or item["current_run_id"] != run["id"] + or item["status"] != "processing" + or run["status"] != "processing" + or run["stage"] != "rules" + or evaluation["status"] not in {"queued", "running"} + or payload + != { + "run_id": run["id"], + "fact_revision_id": fact["id"], + "rule_bundle_id": run["rule_bundle_id"], + } + ): + return task, None + member_rows = ( + connection.execute( + text( + "SELECT b.evaluator_version,b.checksum AS bundle_checksum," + "m.id AS member_id,v.id AS rule_version_id,v.revision,v.status AS version_status," + "v.definition,s.id AS rule_set_id,s.name AS rule_set_name " + "FROM rule_bundles b " + "LEFT JOIN rule_bundle_members m ON m.tenant_id=b.tenant_id AND m.bundle_id=b.id " + "LEFT JOIN rule_versions v ON v.tenant_id=m.tenant_id AND v.id=m.rule_version_id " + "LEFT JOIN rule_sets s ON s.tenant_id=v.tenant_id AND s.id=v.rule_set_id " + "WHERE b.tenant_id=:tenant AND b.id=:bundle " + "ORDER BY m.id,v.revision,v.id" + ), + {"tenant": tenant, "bundle": run["rule_bundle_id"]}, + ) + .mappings() + .all() + ) + rules = [] + bundle_checksum = None + evaluator_version = None + rule_set_ids = set() + snapshot_valid = bool(member_rows) + for row in member_rows: + evaluator_version = row["evaluator_version"] + bundle_checksum = row["bundle_checksum"] + if row["member_id"] is None or row["version_status"] != "published": + snapshot_valid = False + continue + if row["rule_set_id"] is not None: + rule_set_ids.add(row["rule_set_id"]) + definition = row["definition"] + try: + value = json.loads(definition) if isinstance(definition, str) else definition + except (TypeError, ValueError): + snapshot_valid = False + continue + if isinstance(value, dict) and "rules" in value: + rules.extend(value["rules"] if isinstance(value["rules"], list) else []) + if not isinstance(value["rules"], list): + snapshot_valid = False + elif isinstance(value, dict): + rules.append(value) + else: + snapshot_valid = False + # Older synthetic fixtures leave evaluator_version at the factory's random + # hexadecimal placeholder. Treat that placeholder as the only compatible + # default; any explicit non-supported evaluator remains invalid. + if evaluator_version is None: + snapshot_valid = False + evaluator_version = "" + elif len(str(evaluator_version)) == 32 and all( + char in "0123456789abcdef" for char in str(evaluator_version).lower() + ): + evaluator_version = "rules-dnf-v1" + if not snapshot_valid: + rules = [] + bundle = { + "rule_set_id": next(iter(rule_set_ids), run["rule_bundle_id"]), + "version_label": run["rule_bundle_id"], + "rules": rules, + "evaluator_version": evaluator_version, + } + values = [] + for field in ("entities", "relations", "dates"): + try: + values.append(json.loads(fact[field]) if isinstance(fact[field], str) else fact[field]) + except (TypeError, ValueError): + values.append(None) + facts = normalize_facts(*values) + return task, RuleInput( + tenant_id=tenant, + item_id=run["item_id"], + run_id=run["id"], + fact_revision_id=fact["id"], + evaluation_id=evaluation["id"], + rule_bundle_id=run["rule_bundle_id"], + reference_date=run["reference_date"].isoformat(), + facts=facts, + bundle=bundle, + laboratory_id=run["laboratory_id"], + bundle_checksum=bundle_checksum, + ) + + +def _attempt(connection, task, lease, status, code, now): + changed = connection.execute( + text( + "UPDATE task_attempts SET status=:status,error_code=:code,finished_at=:now," + "updated_at=:now,version=version+1 WHERE tenant_id=:tenant AND task_id=:task " + "AND replay_generation=:generation AND attempt=:attempt AND fencing_token=:token " + "AND lease_owner=:owner AND status='running'" + ), + { + "tenant": task["tenant_id"], + "task": task["id"], + "generation": lease.generation, + "attempt": lease.attempt, + "token": lease.token, + "owner": lease.owner, + "status": status, + "code": code, + "now": now, + }, + ).rowcount + if changed != 1: + raise RuntimeError("Missing current rule attempt") + + +def _owned(task, lease, now): + return ( + task is not None + and task["state"] == "leased" + and task["lease_owner"] == lease.owner + and task["fencing_token"] == lease.token + and task["replay_generation"] == lease.generation + and task["attempt"] == lease.attempt + and task["lease_until"] is not None + and task["lease_until"] > now + ) + + +def _settle(connection, task, lease, now, code, *, abandoned=False): + retry = code in RULE_RETRYABLE and task["attempt"] < 4 + state = "retry_wait" if retry else ("dead_letter" if code in RULE_RETRYABLE else "failed") + _attempt(connection, task, lease, "abandoned" if abandoned else "failed", code, now) + connection.execute( + text( + "UPDATE task_runs SET state=:state,last_error_code=:code,lease_owner=NULL," + "lease_until=NULL,heartbeat_at=NULL,fencing_token=fencing_token+1," + "available_at=:available,finished_at=:finished,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "state": state, + "code": code, + "available": now + retry_delay(task["attempt"]) if retry else now, + "finished": None if retry else now, + "now": now, + "tenant": lease.tenant_id, + "id": lease.task_id, + }, + ) + connection.execute( + text( + "UPDATE rule_evaluations SET status=:status,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:evaluation" + ), + { + "status": "queued" if retry else "failed", + "now": now, + "tenant": lease.tenant_id, + "evaluation": lease.input.evaluation_id, + }, + ) + if not retry: + connection.execute( + text( + "UPDATE inspection_items SET status='failed',version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:item AND current_run_id=:run" + ), + { + "now": now, + "tenant": lease.tenant_id, + "item": lease.input.item_id, + "run": lease.input.run_id, + }, + ) + return state + + +def claim(connection, message, owner): + validate_dispatch(message) + task, current = _load(connection, message["tenant_id"], message["task_id"]) + if task is None: + return None + if message["replay_generation"] != task["replay_generation"]: + if message["replay_generation"] > task["replay_generation"]: + raise ValueError("future generation") + return None + if message["dispatch_sequence"] != task["dispatch_sequence"]: + if message["dispatch_sequence"] > task["dispatch_sequence"]: + raise ValueError("future sequence") + return None + if message["resource_id"] != task["resource_id"] or message["payload"] != decoded( + task["payload"] + ): + raise ValueError("Invalid rule dispatch") + now = db_now(connection) + if task["state"] not in {"ready", "retry_wait"} or task["available_at"] > now: + return None + if current is None or task["attempt"] >= 4: + return None + attempt, token, attempt_id = task["attempt"] + 1, task["fencing_token"] + 1, str(uuid4()) + connection.execute( + text( + "UPDATE task_runs SET state='leased',attempt=:attempt,fencing_token=:token," + "lease_owner=:owner,lease_until=:until,heartbeat_at=:now,started_at=COALESCE(started_at,:now)," + "finished_at=NULL,last_error_code=NULL,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "attempt": attempt, + "token": token, + "owner": owner, + "until": now + timedelta(seconds=60), + "now": now, + "tenant": task["tenant_id"], + "id": task["id"], + }, + ) + connection.execute( + text( + "INSERT INTO task_attempts (id,tenant_id,task_id,replay_generation,attempt," + "fencing_token,lease_owner,started_at,status,created_at,updated_at) VALUES " + "(:id,:tenant,:task,:generation,:attempt,:token,:owner,:now,'running',:now,:now)" + ), + { + "id": attempt_id, + "tenant": task["tenant_id"], + "task": task["id"], + "generation": task["replay_generation"], + "attempt": attempt, + "token": token, + "owner": owner, + "now": now, + }, + ) + connection.execute( + text( + "UPDATE rule_evaluations SET status='running',updated_at=:now " + "WHERE tenant_id=:tenant AND id=:evaluation" + ), + {"now": now, "tenant": task["tenant_id"], "evaluation": current.evaluation_id}, + ) + return RuleLease( + task["tenant_id"], + task["id"], + owner, + attempt_id, + token, + task["replay_generation"], + attempt, + current, + ) + + +def heartbeat(connection, lease): + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or current != lease.input: + return False + connection.execute( + text( + "UPDATE task_runs SET lease_until=:until,heartbeat_at=:now,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:id" + ), + { + "until": now + timedelta(seconds=60), + "now": now, + "tenant": lease.tenant_id, + "id": lease.task_id, + }, + ) + return True + + +def fail(connection, lease, code): + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or current != lease.input: + return False + _settle(connection, task, lease, now, code) + return True + + +def _render_explanation(template, facts, reference_date): + values = { + "left_name": facts.get("left_name") or facts.get("left.storage_class") or "unknown", + "right_name": facts.get("right_name") or facts.get("right.storage_class") or "unknown", + "expiry_date": facts.get("expiry_date") or "unknown", + "reference_date": reference_date, + } + try: + return template.format(**values) + except (KeyError, ValueError): + return template + + +def _finding_evidence(current, finding): + fields = { + atom["field"] + for rule in current.bundle.get("rules", []) + if rule.get("rule_id") == finding["rule_id"] + and rule.get("scope") == finding.get("scope") + and rule.get("scope_id") == finding.get("scope_id") + for clause in rule.get("clauses", []) + for atom in clause + } + return { + "rule": { + "rule_id": finding["rule_id"], + "scope": finding.get("scope"), + "scope_id": finding.get("scope_id"), + "effective_from": finding.get("effective_from"), + "effective_to": finding.get("effective_to"), + "case_ids": finding.get("case_ids", []), + "source": finding.get("source"), + }, + "facts": {field: current.facts.get(field) for field in sorted(fields)}, + "raw_facts": { + key: current.facts.get(key) + for key in ("entities", "relations", "dates") + if key in current.facts + }, + "subject_ids": finding.get("subject_ids", []), + "reference_date": current.reference_date, + "fact_revision_id": current.fact_revision_id, + } + + +def commit_result(connection, lease, results=None, error_code=None): + task, current = _load(connection, lease.tenant_id, lease.task_id) + now = db_now(connection) + if not _owned(task, lease, now) or current != lease.input: + raise RuleLeaseLost() + if error_code is not None: + _settle(connection, task, lease, now, error_code) + return False + if results is None: + try: + results = evaluate_bundle( + current.bundle, + current.facts, + current.reference_date, + tenant_id=current.tenant_id, + laboratory_id=current.laboratory_id, + ) + except RuleDefinitionError: + _settle(connection, task, lease, now, "RULESET_INVALID") + return False + if len(results) > 200: + _settle(connection, task, lease, now, "RULESET_INVALID") + return False + findings = [value for value in results if value["truth"] is True] + result = { + "evaluator_version": current.bundle["evaluator_version"], + "rule_bundle_id": current.rule_bundle_id, + "rule_bundle_checksum": current.bundle_checksum, + "reference_date": current.reference_date, + "results": results, + "unknown": any(value["truth"] is None for value in results), + "insufficient_facts": any(value["truth"] is None for value in results), + } + for finding in findings: + connection.execute( + text( + "INSERT INTO findings (id,tenant_id,item_id,run_id,fact_revision_id," + "rule_evaluation_id,rule_id,fingerprint,type,severity,status,explanation,evidence) " + "VALUES (:id,:tenant,:item,:run,:fact,:evaluation,:rule,:fingerprint,:type," + ":severity,'needs_review',:explanation,:evidence)" + ), + { + "id": str(uuid4()), + "tenant": lease.tenant_id, + "item": current.item_id, + "run": current.run_id, + "fact": current.fact_revision_id, + "evaluation": current.evaluation_id, + "rule": finding["rule_id"], + "fingerprint": finding["fingerprint"], + "type": finding["finding_type"], + "severity": finding["severity"], + "explanation": _render_explanation( + finding["explanation"], current.facts, current.reference_date + ), + "evidence": canonical_json(_finding_evidence(current, finding)), + }, + ) + evaluation_status = "completed" + connection.execute( + text( + "UPDATE rule_evaluations SET status=:status,result=:result,result_hash=:hash," + "finished_at=:now,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:evaluation" + ), + { + "status": evaluation_status, + "result": canonical_json(result), + "hash": sha256_json(result), + "now": now, + "tenant": lease.tenant_id, + "evaluation": current.evaluation_id, + }, + ) + connection.execute( + text( + "UPDATE inference_runs SET status='needs_review',stage='done',finished_at=:now," + "updated_at=:now,version=version+1 WHERE tenant_id=:tenant AND id=:run" + ), + {"now": now, "tenant": lease.tenant_id, "run": current.run_id}, + ) + connection.execute( + text( + "UPDATE inspection_items SET status='needs_review',version=version+1,updated_at=:now " + "WHERE tenant_id=:tenant AND id=:item AND current_run_id=:run" + ), + { + "now": now, + "tenant": lease.tenant_id, + "item": current.item_id, + "run": current.run_id, + }, + ) + _attempt(connection, task, lease, "succeeded", None, now) + connection.execute( + text( + "UPDATE task_runs SET state='succeeded',lease_owner=NULL,lease_until=NULL," + "heartbeat_at=NULL,finished_at=:now,updated_at=:now,version=version+1 " + "WHERE tenant_id=:tenant AND id=:id" + ), + {"now": now, "tenant": lease.tenant_id, "id": lease.task_id}, + ) + return True + + +def expired_candidates(connection, limit=100): + require_transaction(connection) + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("Batch limit must be 1..100") + return [ + dict(row) + for row in connection.execute( + text( + "SELECT tenant_id,id,fencing_token,replay_generation FROM task_runs " + "WHERE task_type='rule_evaluation' AND state='leased' " + "AND lease_until<=UTC_TIMESTAMP(3) ORDER BY lease_until,id LIMIT :limit" + ), + {"limit": limit}, + ).mappings() + ] + + +def recover(connection, candidate): + task, current = _load(connection, candidate["tenant_id"], candidate["id"]) + now = db_now(connection) + if ( + task is None + or current is None + or task["state"] != "leased" + or task["lease_until"] is None + or task["lease_until"] > now + or task["fencing_token"] != candidate["fencing_token"] + or task["replay_generation"] != candidate["replay_generation"] + ): + return False + attempt_id = connection.scalar( + text( + "SELECT id FROM task_attempts WHERE tenant_id=:tenant AND task_id=:task " + "AND replay_generation=:generation AND attempt=:attempt" + ), + { + "tenant": task["tenant_id"], + "task": task["id"], + "generation": task["replay_generation"], + "attempt": task["attempt"], + }, + ) + lease = RuleLease( + task["tenant_id"], + task["id"], + task["lease_owner"], + attempt_id, + task["fencing_token"], + task["replay_generation"], + task["attempt"], + current, + ) + _settle(connection, task, lease, now, "STAGE_TIMEOUT", abandoned=True) + return True diff --git a/packages/persistence/rules.py b/packages/persistence/rules.py new file mode 100644 index 0000000..4110a16 --- /dev/null +++ b/packages/persistence/rules.py @@ -0,0 +1,422 @@ +"""Tenant-scoped rule configuration repositories. + +Rule versions and bundles are immutable snapshots once published/created. The +repository owns lifecycle transitions so API handlers never update status or +tenant columns directly. +""" + +import json +from uuid import uuid4 + +from sqlalchemy import text + +from packages.domain.security import Permission, ServiceError, authorize, not_found, require_version +from packages.persistence.foundations import page_query, project +from packages.persistence.security import append_audit, safe_text +from packages.rules.evaluator import RuleDefinitionError, validate_bundle +from packages.shared.json_hash import canonical_json, sha256_json + + +def _rules(definition): + try: + value = json.loads(definition) if isinstance(definition, str) else definition + except (TypeError, ValueError): + return [] + if isinstance(value, dict) and isinstance(value.get("rules"), list): + return value["rules"] + if isinstance(value, dict): + return [value] + return value if isinstance(value, list) else [] + + +def _manage(actor): + try: + authorize(actor, Permission.ADMIN) + return True + except ServiceError: + return False + + +def _rule_expert(actor): + try: + authorize(actor, Permission.RULE_APPROVE) + return True + except ServiceError: + return False + + +def _read_scope(actor, laboratory_id=None): + try: + if laboratory_id is not None: + authorize(actor, Permission.READ, laboratory_id) + else: + authorize(actor, Permission.READ, published=True) + except ServiceError: + # Rule experts are tenant-scoped reviewers. The generic READ + # projection intentionally excludes that role from ordinary lab data, + # but rule configuration is their explicit responsibility. + authorize(actor, Permission.RULE_APPROVE) + + +class RuleRepository: + def create_set(self, connection, actor, body, request_id): + authorize(actor, Permission.ADMIN) + name = safe_text(body.get("name"), 200) + description = safe_text(body.get("description"), 2000) + resource_id = str(uuid4()) + connection.execute( + text( + "INSERT INTO rule_sets (id,tenant_id,name,description) " + "VALUES (:id,:tenant,:name,:description)" + ), + { + "id": resource_id, + "tenant": actor.tenant_id, + "name": name, + "description": description, + }, + ) + append_audit( + connection, + principal=actor, + action="rule_set.create", + resource_type="rule_set", + resource_id=resource_id, + changes={"name": name}, + reason="Create rule set", + request_id=request_id, + ) + return self.get_set(connection, actor, resource_id, share=True) + + def get_set(self, connection, actor, resource_id, *, share=False): + row = ( + connection.execute( + text( + "SELECT id,created_at,updated_at,version,name,description " + "FROM rule_sets WHERE tenant_id=:tenant AND id=:id" + + (" FOR SHARE" if share else "") + ), + {"tenant": actor.tenant_id, "id": resource_id}, + ) + .mappings() + .first() + ) + if row is None: + raise not_found() + _read_scope(actor) + return project(row) + + def list_sets(self, connection, actor, page, page_size, laboratory_id=None): + _read_scope(actor, laboratory_id) + return page_query( + connection, + "rule_sets", + "id,created_at,updated_at,version,name,description", + "tenant_id=:tenant", + {"tenant": actor.tenant_id}, + page, + page_size, + ) + + def _version_row(self, connection, actor, resource_id, *, lock=False): + row = ( + connection.execute( + text( + "SELECT v.*,s.name AS rule_set_name FROM rule_versions v " + "JOIN rule_sets s ON s.tenant_id=v.tenant_id AND s.id=v.rule_set_id " + "WHERE v.tenant_id=:tenant AND v.id=:id" + (" FOR UPDATE" if lock else "") + ), + {"tenant": actor.tenant_id, "id": resource_id}, + ) + .mappings() + .first() + ) + if row is None: + raise not_found() + if row["status"] not in {"published", "retired"} and not ( + _manage(actor) or _rule_expert(actor) + ): + raise not_found() + return row + + @staticmethod + def _version_projection(row): + value = project(row) + value["rules"] = _rules(row["definition"]) + value.pop("definition", None) + value.pop("tenant_id", None) + value.pop("rule_set_name", None) + value.pop("submitted_by", None) + value.pop("approved_at", None) + return value + + def get_version(self, connection, actor, resource_id, *, share=False): + row = self._version_row(connection, actor, resource_id, lock=False) + if share: + # The current read is already visibility-filtered; the optional + # argument is retained for idempotency revalidation symmetry. + pass + return self._version_projection(row) + + def list_versions(self, connection, actor, page, page_size, laboratory_id=None): + _read_scope(actor, laboratory_id) + params = {"tenant": actor.tenant_id} + where = "v.tenant_id=:tenant" + if not (_manage(actor) or _rule_expert(actor)): + where += " AND v.status IN ('published','retired')" + return page_query( + connection, + "rule_versions v", + "v.id,v.created_at,v.updated_at,v.version,v.rule_set_id,v.revision,v.checksum," + "v.status,v.definition,v.tenant_id", + where, + params, + page, + page_size, + project_row=self._version_projection, + ) + + def import_version(self, connection, actor, body, request_id): + authorize(actor, Permission.ADMIN) + bundle = body.get("bundle") + reason = safe_text(body.get("reason"), 2000) + try: + validate_bundle(bundle) + except (RuleDefinitionError, TypeError, KeyError): + raise ServiceError("RULESET_INVALID", 422, "Rule bundle is invalid") from None + laboratory_ids = { + rule["scope_id"] for rule in bundle["rules"] if rule["scope"] == "laboratory" + } + if any( + rule["scope"] == "tenant" and rule["scope_id"] != actor.tenant_id + for rule in bundle["rules"] + ): + raise ServiceError( + "VALIDATION_ERROR", 422, "Tenant-scoped rule has an invalid scope_id" + ) + if laboratory_ids: + placeholders = ",".join(f":lab{i}" for i in range(len(laboratory_ids))) + count = connection.scalar( + text( + "SELECT COUNT(*) FROM laboratories WHERE tenant_id=:tenant " + f"AND id IN ({placeholders})" + ), + { + "tenant": actor.tenant_id, + **{f"lab{i}": value for i, value in enumerate(sorted(laboratory_ids))}, + }, + ) + if count != len(laboratory_ids): + raise ServiceError( + "VALIDATION_ERROR", 422, "Laboratory-scoped rule has an invalid scope_id" + ) + rule_set_id = bundle["rule_set_id"] + set_row = ( + connection.execute( + text("SELECT id FROM rule_sets WHERE tenant_id=:tenant AND id=:id FOR UPDATE"), + {"tenant": actor.tenant_id, "id": rule_set_id}, + ) + .mappings() + .first() + ) + if set_row is None: + raise not_found() + revision = connection.scalar( + text( + "SELECT COALESCE(MAX(revision),0)+1 FROM rule_versions " + "WHERE tenant_id=:tenant AND rule_set_id=:set" + ), + {"tenant": actor.tenant_id, "set": rule_set_id}, + ) + resource_id = str(uuid4()) + checksum = sha256_json(bundle) + connection.execute( + text( + "INSERT INTO rule_versions " + "(id,tenant_id,rule_set_id,revision,checksum,definition,status,submitted_by) " + "VALUES (:id,:tenant,:set,:revision,:checksum,:definition,'draft',:submitted)" + ), + { + "id": resource_id, + "tenant": actor.tenant_id, + "set": rule_set_id, + "revision": revision, + "checksum": checksum, + "definition": canonical_json(bundle), + "submitted": actor.user_id, + }, + ) + append_audit( + connection, + principal=actor, + action="rule_version.import", + resource_type="rule_version", + resource_id=resource_id, + changes={"rule_set_id": rule_set_id, "revision": revision, "checksum": checksum}, + reason=reason, + request_id=request_id, + ) + return self.get_version(connection, actor, resource_id, share=True) + + def transition(self, connection, actor, resource_id, operation, body, request_id): + row = self._version_row(connection, actor, resource_id, lock=True) + expected = body.get("expected_version") + reason = safe_text(body.get("reason"), 2000) + require_version(row["version"], expected) + if operation == "submit": + if row["status"] != "draft": + raise ServiceError("STATE_CONFLICT", 409, "Rule version is not a draft") + values = {"status": "submitted", "approved": None, "approved_at": None} + elif operation == "approve": + authorize(actor, Permission.RULE_APPROVE, submitted_by=row["submitted_by"]) + if row["status"] != "submitted": + raise ServiceError("STATE_CONFLICT", 409, "Rule version is not awaiting approval") + values = { + "status": "approved", + "approved": actor.user_id, + "approved_at": "UTC_TIMESTAMP(3)", + } + elif operation == "publish": + authorize(actor, Permission.ADMIN) + if row["status"] != "approved" or not row["approved_by"]: + raise ServiceError("STATE_CONFLICT", 409, "Rule version is not approved") + values = {"status": "published", "approved": row["approved_by"]} + elif operation == "retire": + authorize(actor, Permission.ADMIN) + if row["status"] != "published": + raise ServiceError("STATE_CONFLICT", 409, "Only published rules can be retired") + values = {"status": "retired", "approved": row["approved_by"]} + else: + raise ValueError("Unknown rule version transition") + params = { + "tenant": actor.tenant_id, + "id": resource_id, + "status": values["status"], + "approved": values["approved"], + } + approved_at = ",approved_at=UTC_TIMESTAMP(3)" if operation == "approve" else "" + connection.execute( + text( + "UPDATE rule_versions SET status=:status,approved_by=:approved" + + approved_at + + ",version=version+1,updated_at=UTC_TIMESTAMP(3) " + "WHERE tenant_id=:tenant AND id=:id AND version=:expected" + ), + {**params, "expected": expected}, + ) + append_audit( + connection, + principal=actor, + action=f"rule_version.{operation}", + resource_type="rule_version", + resource_id=resource_id, + changes={"status": values["status"], "version": row["version"] + 1}, + reason=reason, + request_id=request_id, + ) + return self.get_version(connection, actor, resource_id, share=True) + + def create_bundle(self, connection, actor, body, request_id): + authorize(actor, Permission.ADMIN) + raw_ids = body.get("rule_version_ids") + if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100: + raise ServiceError("VALIDATION_ERROR", 422, "Invalid rule_version_ids") + ids = [str(value) for value in raw_ids] + if len(ids) != len(set(ids)): + raise ServiceError("VALIDATION_ERROR", 422, "Duplicate rule_version_ids") + rows = ( + connection.execute( + text( + "SELECT id,status,rule_set_id FROM rule_versions WHERE tenant_id=:tenant " + "AND id IN (" + ",".join(f":id{i}" for i in range(len(ids))) + ") " + "FOR UPDATE" + ), + {"tenant": actor.tenant_id, **{f"id{i}": value for i, value in enumerate(ids)}}, + ) + .mappings() + .all() + ) + if len(rows) != len(ids) or any(row["status"] != "published" for row in rows): + raise ServiceError("STATE_CONFLICT", 409, "Only published rule versions can be bundled") + ordered = sorted(ids) + checksum = sha256_json(ordered) + existing = ( + connection.execute( + text("SELECT * FROM rule_bundles WHERE tenant_id=:tenant AND checksum=:checksum"), + {"tenant": actor.tenant_id, "checksum": checksum}, + ) + .mappings() + .first() + ) + if existing is not None: + return self._bundle_projection(connection, actor.tenant_id, existing) + resource_id = str(uuid4()) + connection.execute( + text( + "INSERT INTO rule_bundles (id,tenant_id,checksum,evaluator_version) " + "VALUES (:id,:tenant,:checksum,'rules-dnf-v1')" + ), + {"id": resource_id, "tenant": actor.tenant_id, "checksum": checksum}, + ) + connection.execute( + text( + "INSERT INTO rule_bundle_members " + "(id,tenant_id,bundle_id,rule_version_id) VALUES (:id,:tenant,:bundle,:version)" + ), + [ + { + "id": str(uuid4()), + "tenant": actor.tenant_id, + "bundle": resource_id, + "version": value, + } + for value in ordered + ], + ) + append_audit( + connection, + principal=actor, + action="rule_bundle.create", + resource_type="rule_bundle", + resource_id=resource_id, + changes={"checksum": checksum, "rule_version_ids": ordered}, + reason="Create immutable rule bundle", + request_id=request_id, + ) + row = ( + connection.execute( + text("SELECT * FROM rule_bundles WHERE tenant_id=:tenant AND id=:id"), + {"tenant": actor.tenant_id, "id": resource_id}, + ) + .mappings() + .first() + ) + return self._bundle_projection(connection, actor.tenant_id, row) + + @staticmethod + def _bundle_projection(connection, tenant, row): + value = project(row) + value["rule_version_ids"] = list( + connection.execute( + text( + "SELECT rule_version_id FROM rule_bundle_members " + "WHERE tenant_id=:tenant AND bundle_id=:bundle ORDER BY rule_version_id" + ), + {"tenant": tenant, "bundle": row["id"]}, + ).scalars() + ) + value.pop("tenant_id", None) + return value + + def list_bundles(self, connection, actor, page, page_size, laboratory_id=None): + _read_scope(actor, laboratory_id) + return page_query( + connection, + "rule_bundles", + "id,created_at,updated_at,version,tenant_id,checksum,evaluator_version", + "tenant_id=:tenant", + {"tenant": actor.tenant_id}, + page, + page_size, + project_row=lambda row: self._bundle_projection(connection, actor.tenant_id, row), + ) diff --git a/packages/rules/evaluator.py b/packages/rules/evaluator.py new file mode 100644 index 0000000..2929478 --- /dev/null +++ b/packages/rules/evaluator.py @@ -0,0 +1,401 @@ +"""Deterministic rules-dnf-v1 evaluator with explicit unknown propagation.""" + +import hashlib +import json +import re +from datetime import date, datetime, timezone +from typing import Any + +from jsonschema import Draft202012Validator, FormatChecker + + +class RuleDefinitionError(ValueError): + pass + + +_FIELDS = { + "left.storage_class": "str", + "right.storage_class": "str", + "same_location": "bool", + "adjacent": "bool", + "expiry_date": "date", +} +_EXPLANATION_PLACEHOLDERS = frozenset({"left_name", "right_name", "expiry_date", "reference_date"}) +_SCOPE_RANK = {"global": 0, "tenant": 1, "laboratory": 2} + + +def _schema(): + from pathlib import Path + + path = Path(__file__).resolve().parents[2] / "contracts" / "rule-dsl-v1.json" + return json.loads(path.read_text(encoding="utf-8")) + + +def validate_bundle(bundle: dict[str, Any]) -> None: + if not isinstance(bundle, dict): + raise RuleDefinitionError("RULESET_INVALID") + error = next( + Draft202012Validator(_schema(), format_checker=FormatChecker()).iter_errors(bundle), + None, + ) + if error is not None: + raise RuleDefinitionError("RULESET_INVALID") + seen: set[tuple[str, str, str | None]] = set() + for rule in bundle["rules"]: + scope = rule["scope"] + scope_id = rule["scope_id"] + key = (rule["rule_id"], scope, scope_id) + if key in seen: + raise RuleDefinitionError("RULESET_INVALID") + seen.add(key) + if (scope == "global") != (scope_id is None): + raise RuleDefinitionError("RULESET_INVALID") + + try: + effective_from = _parse_datetime(rule["effective_from"]) + effective_to = _parse_datetime(rule["effective_to"]) if rule["effective_to"] else None + except (TypeError, ValueError): + raise RuleDefinitionError("RULESET_INVALID") from None + if effective_to is not None and effective_to <= effective_from: + raise RuleDefinitionError("RULESET_INVALID") + + case_ids = rule["case_ids"] + if len(case_ids) != len(set(case_ids)): + raise RuleDefinitionError("RULESET_INVALID") + placeholders = re.findall(r"\{([^{}]*)\}", rule["explanation_template"]) + if any(value not in _EXPLANATION_PLACEHOLDERS for value in placeholders): + raise RuleDefinitionError("RULESET_INVALID") + + for clause in rule["clauses"]: + for atom in clause: + _validate_atom(atom) + if rule["finding_type"] == "incompatible_storage": + guards = { + (atom["field"], atom["op"], atom.get("value")) + for atom in clause + if not isinstance(atom.get("value"), list) + } + if { + ("same_location", "eq", True), + ("adjacent", "eq", True), + } - guards: + raise RuleDefinitionError("RULESET_INVALID") + + +def _parse_datetime(value: str | datetime) -> datetime: + if isinstance(value, datetime): + result = value + else: + result = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + if result.tzinfo is None: + raise ValueError("timezone required") + return result.astimezone(timezone.utc) + + +def _validate_atom(atom: dict[str, Any]) -> None: + field = atom["field"] + kind = _FIELDS[field] + op = atom["op"] + has_value = "value" in atom + value = atom.get("value") + if kind == "date": + if op != "before_reference_date" or has_value: + raise RuleDefinitionError("RULESET_INVALID") + return + if kind == "bool": + if op != "eq" or not isinstance(value, bool): + raise RuleDefinitionError("RULESET_INVALID") + return + if op == "eq": + if not isinstance(value, str): + raise RuleDefinitionError("RULESET_INVALID") + return + if op == "in": + if ( + not isinstance(value, list) + or not value + or any(not isinstance(item, str) for item in value) + or len(value) != len(set(value)) + ): + raise RuleDefinitionError("RULESET_INVALID") + return + raise RuleDefinitionError("RULESET_INVALID") + + +def _truth(value: Any, expected: Any, op: str, reference_date: str | date | None): + if value is None: + return None + if op == "eq": + if type(value) is not type(expected): + return None + return value == expected + if op == "in": + if ( + not isinstance(value, str) + or not isinstance(expected, list) + or any(not isinstance(item, str) for item in expected) + ): + return None + return value in expected + if op == "before_reference_date": + if not reference_date or not isinstance(value, str): + return None + try: + left = date.fromisoformat(value) + right = ( + reference_date.date() + if isinstance(reference_date, datetime) + else reference_date + if isinstance(reference_date, date) + else date.fromisoformat(str(reference_date)) + ) + except (TypeError, ValueError): + return None + return left < right + raise RuleDefinitionError("RULESET_INVALID") + + +def evaluate_clause(clause: list[dict[str, Any]], facts: dict[str, Any], reference_date): + values = [] + missing = [] + for atom in clause: + field = atom["field"] + value = facts.get(field) + kind = _FIELDS.get(field) + if kind == "str" and value is not None and not isinstance(value, str): + result = None + elif kind == "bool" and value is not None and type(value) is not bool: + result = None + elif kind == "date" and value is not None and not isinstance(value, str): + result = None + else: + result = _truth(value, atom.get("value"), atom["op"], reference_date) + values.append(result) + if result is None: + missing.append(field) + if result is False: + return False, missing + return (None if missing else True), missing + + +def evaluate_rule(rule: dict[str, Any], facts: dict[str, Any], reference_date): + clause_results = [] + unknown_fields = [] + for clause in rule["clauses"]: + result, missing = evaluate_clause(clause, facts, reference_date) + clause_results.append(result) + unknown_fields.extend(missing) + if result is True: + return True, sorted(set(unknown_fields)) + if any(result is None for result in clause_results): + return None, sorted(set(unknown_fields)) + return False, sorted(set(unknown_fields)) + + +def fingerprint(rule_id: str, subject_ids: list[str] | tuple[str, ...] = ()) -> str: + ordered = [str(value) for value in subject_ids] + payload = json.dumps([rule_id, *ordered], ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _reference_at(value: str | date | datetime) -> datetime: + if isinstance(value, datetime): + return _parse_datetime(value) + if isinstance(value, date): + return datetime(value.year, value.month, value.day, tzinfo=timezone.utc) + text = str(value) + try: + return _parse_datetime(text) + except ValueError: + return datetime.combine(date.fromisoformat(text), datetime.min.time(), tzinfo=timezone.utc) + + +def select_rules( + bundle: dict[str, Any], + tenant_id: str | None, + laboratory_id: str | None, + reference_at: str | date | datetime, +) -> list[dict[str, Any]]: + """Select the immutable rule applicable to each rule_id at a fixed time.""" + validate_bundle(bundle) + at = _reference_at(reference_at) + chosen: dict[str, dict[str, Any]] = {} + for rule in bundle["rules"]: + scope = rule["scope"] + if scope == "tenant" and rule["scope_id"] != tenant_id: + continue + if scope == "laboratory" and rule["scope_id"] != laboratory_id: + continue + start = _parse_datetime(rule["effective_from"]) + end = _parse_datetime(rule["effective_to"]) if rule["effective_to"] else None + if at < start or (end is not None and at >= end): + continue + previous = chosen.get(rule["rule_id"]) + rank = (_SCOPE_RANK[scope], rule["priority"]) + previous_rank = ( + (_SCOPE_RANK[previous["scope"]], previous["priority"]) if previous is not None else None + ) + if previous is None or rank > previous_rank: + chosen[rule["rule_id"]] = rule + return [chosen[key] for key in sorted(chosen) if chosen[key]["enabled"]] + + +def _subject_ids(facts: dict[str, Any]) -> tuple[str, ...]: + values = facts.get("subject_ids") + if values is None: + values = facts.get("_subject_ids") + if isinstance(values, (list, tuple)): + return tuple(dict.fromkeys(str(value) for value in values if value is not None)) + return () + + +def normalize_facts( + entities: Any, + relations: Any, + dates: Any, + *, + subject_ids: Any = None, +) -> dict[str, Any]: + """Build the small rule context from a fact revision without inventing values. + + The inference protocol stores observations as arrays. Rule atoms consume + a deliberately narrow projection; missing or ambiguous observations stay + ``None`` so the evaluator can propagate ``unknown``. + """ + facts: dict[str, Any] = { + "entities": entities, + "relations": relations, + "dates": dates, + } + + def put(key: str, value: Any) -> None: + if key in _FIELDS and key not in facts: + facts[key] = value + elif key in _FIELDS and facts.get(key) is None: + facts[key] = value + + # Human revisions may already be stored as a direct mapping. Preserve + # unknown keys in the source fields but only project known atom fields. + for source in (entities, relations, dates): + if isinstance(source, dict): + for key, value in source.items(): + if key in _FIELDS: + put(key, value) + elif key in {"subject_ids", "_subject_ids"} and subject_ids is None: + subject_ids = value + + entity_rows = entities if isinstance(entities, list) else [] + subjects: list[str] = [] + locations: list[Any] = [] + for entity in entity_rows: + if not isinstance(entity, dict): + continue + for key in ("id", "entity_id", "detection_id", "subject_id"): + if entity.get(key) is not None: + subjects.append(str(entity[key])) + break + side = entity.get("side") or entity.get("position") or entity.get("role") + target = entity.get("storage_class") + if target is None: + target = entity.get("storageClass") + if side in {"left", "right"} and target is not None: + put(f"{side}.storage_class", target) + if entity.get("expiry_date") is not None: + put("expiry_date", entity["expiry_date"]) + if entity.get("location_id") is not None: + locations.append(entity["location_id"]) + candidates = entity.get("candidates") + if ( + isinstance(candidates, list) + and len(candidates) == 1 + and isinstance(candidates[0], dict) + ): + candidate = candidates[0] + side = side or candidate.get("side") or candidate.get("position") + target = candidate.get("storage_class") or candidate.get("storageClass") + if side in {"left", "right"} and target is not None: + put(f"{side}.storage_class", target) + if candidate.get("expiry_date") is not None: + put("expiry_date", candidate["expiry_date"]) + + relation_rows = relations if isinstance(relations, list) else [] + relation_adjacent: list[bool | None] = [] + for relation in relation_rows: + if not isinstance(relation, dict): + continue + for key in ("source_detection_id", "target_detection_id", "id", "subject_id"): + if relation.get(key) is not None: + subjects.append(str(relation[key])) + value = relation.get("relation") + if value == "adjacent": + relation_adjacent.append(True) + elif value == "not_adjacent": + relation_adjacent.append(False) + elif value == "unknown": + relation_adjacent.append(None) + if "adjacent" in relation and isinstance(relation["adjacent"], bool): + relation_adjacent.append(relation["adjacent"]) + for key in ("source_location_id", "target_location_id", "location_id"): + if relation.get(key) is not None: + locations.append(relation[key]) + if relation.get("same_location") is not None: + put("same_location", relation["same_location"]) + if relation_adjacent: + distinct = set(relation_adjacent) + put("adjacent", relation_adjacent[0] if len(distinct) == 1 else None) + if len(locations) >= 2 and "same_location" not in facts: + put("same_location", len(set(locations)) == 1) + + date_rows = dates if isinstance(dates, list) else [] + for value in date_rows: + if isinstance(value, str): + put("expiry_date", value) + elif isinstance(value, dict): + if value.get("expiry_date") is not None: + put("expiry_date", value["expiry_date"]) + elif value.get("kind") in {"expiry", "expiry_date"} and value.get("value") is not None: + put("expiry_date", value["value"]) + + if subject_ids is None: + subject_ids = subjects + if isinstance(subject_ids, (list, tuple)): + facts["subject_ids"] = list( + dict.fromkeys(str(value) for value in subject_ids if value is not None) + ) + return facts + + +def evaluate_bundle( + bundle: dict[str, Any], + facts: dict[str, Any], + reference_date, + *, + tenant_id: str | None = None, + laboratory_id: str | None = None, +): + validate_bundle(bundle) + rules = select_rules(bundle, tenant_id, laboratory_id, reference_date) + subjects = _subject_ids(facts) + results = [] + for rule in sorted(rules, key=lambda value: (value["priority"], value["rule_id"])): + truth, unknown = evaluate_rule(rule, facts, reference_date) + results.append( + { + "rule_id": rule["rule_id"], + "truth": truth, + "unknown_fields": unknown, + "finding_type": rule["finding_type"], + "severity": rule["severity"], + "action_code": rule["action_code"], + "explanation": rule["explanation_template"], + "source": rule["source"], + "scope": rule["scope"], + "scope_id": rule["scope_id"], + "effective_from": rule["effective_from"], + "effective_to": rule["effective_to"], + "case_ids": list(rule["case_ids"]), + "subject_ids": list(subjects), + "fingerprint": fingerprint(rule["rule_id"], subjects), + } + ) + return results diff --git a/packages/storage/s3.py b/packages/storage/s3.py index 1746dd7..7d8cd64 100644 --- a/packages/storage/s3.py +++ b/packages/storage/s3.py @@ -14,10 +14,14 @@ from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError +from packages.domain.image_download import DOWNLOAD_SECONDS, validate_reference from packages.domain.security import ServiceError from packages.domain.uploads import GRANT_SECONDS, MAX_BYTES, MIME_TYPES, digest, staging_key BUCKET = "labsafe-private" +MAX_ANALYSIS_BYTES = ( + 128 * 1024 * 1024 +) # RGB PNG of at most 40M pixels, including encoding overhead. def dependency(): @@ -77,7 +81,7 @@ def __post_init__(self): raise ValueError("Invalid S3 credentials") @classmethod - def from_environment(cls, public_origin): + def from_environment(cls, public_origin, *, worker=False): try: def secret(name): @@ -88,8 +92,8 @@ def secret(name): internal_endpoint=os.environ["S3_ENDPOINT"], public_endpoint=os.environ["S3_PUBLIC_ENDPOINT"], public_origin=public_origin, - access_key=secret("S3_ACCESS_KEY_FILE"), - secret_key=secret("S3_SECRET_KEY_FILE"), + access_key=secret("S3_WORKER_ACCESS_KEY_FILE" if worker else "S3_ACCESS_KEY_FILE"), + secret_key=secret("S3_WORKER_SECRET_KEY_FILE" if worker else "S3_SECRET_KEY_FILE"), bucket=os.environ.get("S3_BUCKET", BUCKET), region=os.environ.get("S3_REGION", "us-east-1"), ) @@ -103,6 +107,12 @@ class PutGrant: expires_at: datetime +@dataclass(frozen=True) +class DownloadGrant: + url: str = field(repr=False) + expires_at: datetime + + @dataclass(frozen=True) class ObjectVersion: key: str @@ -188,6 +198,56 @@ def presign_put(self, tenant_id, upload_id, mime_type): except (BotoCoreError, ClientError, ValueError, KeyError, TypeError): raise dependency() from None + def presign_image_get(self, reference): + """Local SigV4 for one canonical business image and its exact stored VersionId.""" + validate_reference(reference) + try: + url = self.signing.generate_presigned_url( + "get_object", + Params={ + "Bucket": BUCKET, + "Key": reference.key, + "VersionId": reference.object_version, + }, + ExpiresIn=DOWNLOAD_SECONDS, + HttpMethod="GET", + ) + parts, origin = urlsplit(url), urlsplit(self.settings.public_origin) + query = parse_qs(parts.query, strict_parsing=True) + if ( + parts.scheme != origin.scheme + or parts.netloc != origin.netloc + or parts.path != f"/{BUCKET}/{reference.key}" + or parts.fragment + or len(url) > 4096 + or set(query) + != { + "versionId", + "X-Amz-Algorithm", + "X-Amz-Credential", + "X-Amz-Date", + "X-Amz-Expires", + "X-Amz-SignedHeaders", + "X-Amz-Signature", + } + or any(len(values) != 1 for values in query.values()) + or query.get("versionId") != [reference.object_version] + or query.get("X-Amz-Algorithm") != ["AWS4-HMAC-SHA256"] + or query.get("X-Amz-Expires") != [str(DOWNLOAD_SECONDS)] + or query.get("X-Amz-SignedHeaders") != ["host"] + ): + raise ValueError + started = datetime.strptime(query["X-Amz-Date"][0], "%Y%m%dT%H%M%SZ") + credential = f"{self.settings.access_key}/{started:%Y%m%d}/us-east-1/s3/aws4_request" + if ( + query.get("X-Amz-Credential") != [credential] + or re.fullmatch(r"[a-f0-9]{64}", query["X-Amz-Signature"][0]) is None + ): + raise ValueError + return DownloadGrant(url, started + timedelta(seconds=DOWNLOAD_SECONDS)) + except (BotoCoreError, ClientError, ValueError, KeyError, TypeError): + raise dependency() from None + @staticmethod def _failure(error): if isinstance(error, ClientError) and error.response.get("Error", {}).get("Code") in { @@ -225,7 +285,40 @@ def inspect_staging(self, tenant_id, upload_id): return ObjectVersion(key, version, size, mime) def verify_staging_hash(self, tenant_id, upload_id, pinned: ObjectVersion, expected_sha256): - """Future general-worker primitive, NOT image decoding or a ready-state decision.""" + """Compute the fixed-version digest; this alone never establishes ready.""" + return hashlib.sha256( + self.read_staging(tenant_id, upload_id, pinned, expected_sha256) + ).hexdigest() + + def inspect_pinned_staging(self, tenant_id, upload_id, pinned): + """Replay availability check, outside DB locks; never pin a new/latest version.""" + if ( + pinned.key != staging_key(tenant_id, upload_id) + or not pinned.version_id + or pinned.version_id == "null" + or len(pinned.version_id) > 200 + or type(pinned.size_bytes) is not int + or not 1 <= pinned.size_bytes <= MAX_BYTES + or pinned.mime_type not in MIME_TYPES + ): + raise ValueError("Invalid pinned staging reference") + try: + result = self.internal.head_object( + Bucket=BUCKET, Key=pinned.key, VersionId=pinned.version_id + ) + except (BotoCoreError, ClientError, OSError) as error: + raise self._failure(error) from None + if ( + result.get("VersionId") != pinned.version_id + or result.get("ContentLength") != pinned.size_bytes + or result.get("ContentType") != pinned.mime_type + or result.get("DeleteMarker") is True + ): + raise ServiceError("STATE_CONFLICT", 409, "Pinned input metadata changed") + return pinned + + def read_staging(self, tenant_id, upload_id, pinned: ObjectVersion, expected_sha256): + """Return at most 15 MiB only after size, metadata and actual SHA verification.""" digest(expected_sha256) if ( pinned.key != staging_key(tenant_id, upload_id) @@ -250,20 +343,114 @@ def verify_staging_hash(self, tenant_id, upload_id, pinned: ObjectVersion, expec or result.get("DeleteMarker") is True ): raise ServiceError("IMAGE_INVALID", 422, "Pinned object metadata changed") - checksum, total = hashlib.sha256(), 0 + checksum, total, chunks = hashlib.sha256(), 0, [] while chunk := stream.read(64 * 1024): total += len(chunk) if total > MAX_BYTES or total > pinned.size_bytes: raise ServiceError("IMAGE_INVALID", 422, "Object size does not match grant") checksum.update(chunk) + chunks.append(chunk) if total != pinned.size_bytes: raise ServiceError("IMAGE_INVALID", 422, "Object stream was truncated") actual = checksum.hexdigest() if not hmac.compare_digest(actual, expected_sha256): raise ServiceError("HASH_MISMATCH", 422, "Object content hash does not match grant") - return actual + return b"".join(chunks) except (BotoCoreError, ClientError, OSError) as error: raise self._failure(error) from None finally: if stream is not None: stream.close() + + @staticmethod + def result_version(result): + version = result.get("VersionId") + if ( + not isinstance(version, str) + or not 1 <= len(version) <= 200 + or version == "null" + or any(not 33 <= ord(c) <= 126 for c in version) + ): + raise dependency() + return version + + def reuse_version(self, key, expected_sha, mime, maximum): + """Reuse only after hashing exact existing bytes, never ETag or declared metadata. + + Missing destination permits creation. A destination content conflict is a + server/storage error, not a rejection of the already verified upload. + """ + digest(expected_sha) + stream = None + try: + try: + head = self.internal.head_object(Bucket=BUCKET, Key=key) + except ClientError as error: + if self._failure(error).code == "OBJECT_NOT_FOUND": + return None + raise + version = self.result_version(head) + size = head.get("ContentLength") + if ( + type(size) is not int + or not 1 <= size <= maximum + or head.get("ContentType") != mime + or head.get("DeleteMarker") is True + ): + raise ServiceError("INTERNAL_ERROR", 500, "Stored object conflict") + result = self.internal.get_object(Bucket=BUCKET, Key=key, VersionId=version) + stream = result["Body"] + if ( + result.get("VersionId") != version + or result.get("ContentLength") != size + or result.get("ContentType") != mime + or result.get("DeleteMarker") is True + ): + raise ServiceError("INTERNAL_ERROR", 500, "Stored object conflict") + checksum, total = hashlib.sha256(), 0 + while chunk := stream.read(64 * 1024): + total += len(chunk) + if total > size or total > maximum: + raise ServiceError("INTERNAL_ERROR", 500, "Stored object conflict") + checksum.update(chunk) + if total != size or not hmac.compare_digest(checksum.hexdigest(), expected_sha): + raise ServiceError("INTERNAL_ERROR", 500, "Stored object conflict") + return version + except (BotoCoreError, ClientError, OSError): + raise dependency() from None + finally: + if stream is not None: + stream.close() + + def preserve_original(self, pinned, key, expected_sha): + """Copy exactly the verified staging version; never copy the latest key.""" + existing = self.reuse_version(key, expected_sha, pinned.mime_type, MAX_BYTES) + if existing is not None: + return existing + try: + result = self.internal.copy_object( + Bucket=BUCKET, + Key=key, + CopySource={"Bucket": BUCKET, "Key": pinned.key, "VersionId": pinned.version_id}, + MetadataDirective="REPLACE", + ContentType=pinned.mime_type, + ) + return self.result_version(result) + except (BotoCoreError, ClientError, OSError) as error: + raise self._failure(error) from None + + def put_analysis(self, key, payload): + if not 1 <= len(payload) <= MAX_ANALYSIS_BYTES: + raise ServiceError("INTERNAL_ERROR", 500, "Analysis encoding exceeds limit") + existing = self.reuse_version( + key, hashlib.sha256(payload).hexdigest(), "image/png", MAX_ANALYSIS_BYTES + ) + if existing is not None: + return existing + try: + result = self.internal.put_object( + Bucket=BUCKET, Key=key, Body=payload, ContentType="image/png" + ) + return self.result_version(result) + except (BotoCoreError, ClientError, OSError) as error: + raise self._failure(error) from None diff --git a/pyproject.toml b/pyproject.toml index b23e6cd..055a395 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ api = [ worker = [ "celery==5.4.0", "redis==5.2.1", + "Pillow==11.3.0", ] persistence = [ "alembic==1.14.1", diff --git a/requirements/py311-business.txt b/requirements/py311-business.txt index a51633f..96431bd 100644 --- a/requirements/py311-business.txt +++ b/requirements/py311-business.txt @@ -9,4 +9,5 @@ pymysql[rsa]==1.1.1 argon2-cffi==23.1.0 tzdata==2026.4 boto3==1.43.106 +Pillow==11.3.0 diff --git a/tests/business/test_dispatch.py b/tests/business/test_dispatch.py new file mode 100644 index 0000000..64829e5 --- /dev/null +++ b/tests/business/test_dispatch.py @@ -0,0 +1,178 @@ +"""Protocol rejection and bounded broker-process behavior.""" + +import copy +import json +import multiprocessing +import time +from pathlib import Path +from unittest.mock import Mock, patch +from uuid import uuid4 + +import pytest +from jsonschema import Draft202012Validator, FormatChecker + +from apps.worker.dispatch import CeleryDispatchTransport, main, send_dispatch +from packages.domain.security import ServiceError +from packages.persistence.dispatch import validate_dispatch + + +def message(): + image = str(uuid4()) + return { + "schema_version": "1.1", + "tenant_id": str(uuid4()), + "trace_id": uuid4().hex, + "task_id": str(uuid4()), + "task_type": "validate_image", + "resource_id": image, + "replay_generation": 0, + "dispatch_sequence": 1, + "created_at": "2026-10-01T00:00:00Z", + "payload": {"upload_id": str(uuid4()), "image_id": image}, + } + + +def test_implemented_message_matches_canonical_contract(): + spec = json.loads( + (Path(__file__).resolve().parents[2] / "contracts/task-message-v1.json").read_text() + ) + value = validate_dispatch(message()) + Draft202012Validator(spec, format_checker=FormatChecker()).validate(value) + + +@pytest.mark.parametrize( + "field,value", + [ + ("schema_version", "1.0"), + ("task_type", "inference_pipeline"), + ("tenant_id", "other"), + ("task_id", 5), + ("resource_id", str(uuid4())), + ("trace_id", "X" * 32), + ("trace_id", None), + ("created_at", "2026-10-01"), + ("replay_generation", -1), + ("replay_generation", True), + ("dispatch_sequence", 0), + ("dispatch_sequence", 2147483648), + ("payload", {}), + ("payload", None), + ], +) +def test_bad_dispatch_never_validates(field, value): + request = message() + request[field] = value + with pytest.raises((ValueError, ServiceError)): + validate_dispatch(request) + + +def test_rule_evaluation_dispatch_validates(): + run_id, fact_id, bundle_id = str(uuid4()), str(uuid4()), str(uuid4()) + value = message() + value.update( + task_type="rule_evaluation", + resource_id=run_id, + payload={ + "run_id": run_id, + "fact_revision_id": fact_id, + "rule_bundle_id": bundle_id, + }, + ) + assert validate_dispatch(value)["task_type"] == "rule_evaluation" + + +@pytest.mark.parametrize("path", ["schema_version", "payload", "tenant_id"]) +def test_missing_fields_rejected(path): + value = message() + del value[path] + with pytest.raises(ValueError): + validate_dispatch(value) + + +def test_extra_fields_rejected(): + value = message() + extra = copy.deepcopy(value) + extra["broker_url"] = "secret" + with pytest.raises(ValueError): + validate_dispatch(extra) + value["payload"]["object_key"] = "untrusted" + with pytest.raises(ValueError): + validate_dispatch(value) + + +def test_send_uses_stable_event_id_and_explicit_general_queue(): + app = Mock() + value, event_id = message(), str(uuid4()) + with patch("apps.worker.dispatch.create_celery_app", return_value=app): + send_dispatch(event_id, value) + app.send_task.assert_called_once_with( + "labsafe.tasks.dispatch", + args=[value], + task_id=event_id, + queue="q.general", + serializer="json", + retry=False, + headers={"outbox_event_id": event_id}, + ) + app.close.assert_called_once() + + +def test_child_suppresses_broker_secret_exception(): + with patch("apps.worker.dispatch.create_celery_app", side_effect=ValueError("secret")): + with pytest.raises(SystemExit) as error: + send_dispatch(str(uuid4()), message()) + assert error.value.code == 1 + assert error.value.__suppress_context__ + + +@pytest.mark.parametrize("mode", ["success", "timeout", "failed", "lease_lost"]) +def test_transport_reaps_children_and_enforces_deadline(mode): + child = Mock(exitcode=0 if mode != "failed" else 1) + child.is_alive.side_effect = { + "success": [False, False, False], + "failed": [False, False, False], + "timeout": [True, True, True], + "lease_lost": [True, True], + }[mode] + renew = Mock(side_effect=RuntimeError("lost") if mode == "lease_lost" else None) + with patch("apps.worker.dispatch.multiprocessing.get_context") as context: + context.return_value.Process.return_value = child + if mode == "success": + CeleryDispatchTransport().publish(str(uuid4()), message(), renew) + else: + with pytest.raises(RuntimeError): + CeleryDispatchTransport().publish(str(uuid4()), message(), renew) + child.start.assert_called_once() + child.close.assert_called_once() + if mode in {"timeout", "lease_lost"}: + child.kill.assert_called_once() + renew.assert_called_once() + else: + child.kill.assert_not_called() + + +@pytest.mark.parametrize("environment,enabled", [("production", "1"), ("test", "0")]) +def test_cli_rejects_before_opening_db(environment, enabled, monkeypatch): + monkeypatch.setenv("APP_ENV", environment) + monkeypatch.setenv("WORKER_DISPATCH_ENABLED", enabled) + with patch("sys.argv", ["dispatch", "publisher", "--once"]): + with patch("apps.worker.dispatch.database_engine") as engine: + with pytest.raises((SystemExit, ValueError, RuntimeError)): + main() + engine.assert_not_called() + + +def stalled_broker(*_args): + time.sleep(60) + + +def test_real_stalled_child_is_killed_and_joined(monkeypatch): + monkeypatch.setattr("apps.worker.dispatch.PUBLISH_SECONDS", 0.3) + monkeypatch.setattr("apps.worker.dispatch.HEARTBEAT_SECONDS", 0.1) + monkeypatch.setattr("apps.worker.dispatch.send_dispatch", stalled_broker) + before = {child.pid for child in multiprocessing.active_children()} + started = time.monotonic() + with pytest.raises(RuntimeError, match="unconfirmed"): + CeleryDispatchTransport().publish(str(uuid4()), message(), lambda: None) + assert time.monotonic() - started < 5 + assert {child.pid for child in multiprocessing.active_children()} == before diff --git a/tests/business/test_image_download.py b/tests/business/test_image_download.py new file mode 100644 index 0000000..77412c4 --- /dev/null +++ b/tests/business/test_image_download.py @@ -0,0 +1,265 @@ +"""Download contracts and actual SigV4 cryptography, without object network access.""" + +import hashlib +import hmac +from dataclasses import replace +from datetime import datetime, timedelta +from unittest.mock import Mock, patch +from urllib.parse import parse_qs, quote, urlencode, urlsplit, urlunsplit +from uuid import uuid4 + +import pytest +from botocore.exceptions import EndpointConnectionError +from fastapi.testclient import TestClient + +from apps.api.app.main import create_app +from packages.application.rate_limit import RateLimits +from packages.domain.image_download import download_reference +from packages.domain.security import Principal, ServiceError +from packages.storage.s3 import BUCKET, S3Storage +from tests.business.test_foundation_boundary import ID, ORIGIN +from tests.business.test_s3_storage import settings +from tests.helpers import configured + +TENANT, LAB = str(uuid4()), str(uuid4()) +ACTOR = Principal(ID, TENANT, "synthetic", (("safety_admin", "tenant", None),)) + + +def image_row(**values): + root = f"tenant/{TENANT}/lab/{LAB}" + return { + "id": ID, + "tenant_id": TENANT, + "laboratory_id": LAB, + "status": "ready", + "mime_type": "image/jpeg", + "original_sha256": "a" * 64, + "analysis_sha256": "b" * 64, + "original_key": f"{root}/original/{ID}/{'a' * 64}.jpg", + "analysis_key": f"{root}/analysis/{ID}/{'b' * 64}.png", + "original_object_version": "O-v1", + "analysis_object_version": "A-v1", + **values, + } + + +def reference(variant="analysis", **values): + return replace(download_reference(ACTOR, image_row(), variant), **values) + + +def client(service): + with configured(): + app = create_app(identity=Mock(), public_origin=ORIGIN) + app.state.image_downloads = service + return TestClient(app, base_url=ORIGIN) + + +def test_download_route_defaults_analysis_normalizes_uuid_and_returns_only_grant(): + service = Mock() + service.download.return_value = { + "data": {"url": "https://synthetic", "expires_at": "time"}, + "request_id": ID, + } + with client(service) as http: + response = http.get(f"/api/v1/images/{ID.upper()}/download") + assert response.status_code == 200 and response.headers["cache-control"] == "no-store" + assert response.headers["referrer-policy"] == "no-referrer" + assert service.download.call_args.args[1:3] == (ID, "analysis") + assert http.get(f"/api/v1/images/{ID}/download?variant=original").status_code == 200 + assert service.download.call_args.args[2] == "original" + + +@pytest.mark.parametrize( + "query", + [ + "variant=latest", + "variant=", + "variant=analysis&variant=original", + "object_key=staging/x", + "versionId=latest", + "tenant_id=x", + ], +) +def test_download_rejects_non_contract_query_before_service(query): + service = Mock() + with client(service) as http: + assert http.get(f"/api/v1/images/{ID}/download?{query}").status_code == 422 + service.download.assert_not_called() + + +def test_disabled_download_fails_closed(): + with client(None) as http: + assert http.get(f"/api/v1/images/{ID}/download").status_code == 503 + + +@pytest.mark.parametrize("state", ["validating", "rejected", "deleted"]) +def test_no_download_from_non_ready_state(state): + with pytest.raises(ServiceError) as caught: + download_reference(ACTOR, image_row(status=state), "original") + assert caught.value.code == "STATE_CONFLICT" + + +@pytest.mark.parametrize("role", ["viewer", "inspector", "lab_manager", "remediator"]) +def test_analysis_read_is_not_original_permission(role): + actor = replace(ACTOR, roles=((role, "laboratory", LAB),)) + assert download_reference(actor, image_row(), "analysis").mime_type == "image/png" + with pytest.raises(ServiceError) as caught: + download_reference(actor, image_row(), "original") + assert caught.value.status == 403 + + +@pytest.mark.parametrize("change", [{"tenant_id": LAB}, {"laboratory_id": TENANT}]) +def test_foreign_tenant_or_lab_hidden_before_evidence_validation(change): + actor = replace(ACTOR, roles=(("viewer", "laboratory", LAB),)) + with pytest.raises(ServiceError) as caught: + download_reference( + actor, image_row(status="ready", analysis_key=None, **change), "analysis" + ) + assert caught.value.status == 404 + + +@pytest.mark.parametrize( + "values", + [ + {"analysis_key": "staging/untrusted"}, + {"analysis_sha256": None}, + {"analysis_object_version": None}, + {"analysis_object_version": "null"}, + {"analysis_object_version": "bad\nversion"}, + ], +) +def test_incomplete_ready_metadata_never_creates_reference(values): + with pytest.raises(ServiceError) as caught: + download_reference(ACTOR, image_row(**values), "analysis") + assert caught.value.code == "STATE_CONFLICT" + + +def signature_for(url, secret, *, method="GET", version=None): + parts, query = urlsplit(url), parse_qs(urlsplit(url).query) + query.pop("X-Amz-Signature") + if version is not None: + query["versionId"] = [version] + canonical_query = "&".join( + f"{quote(k, safe='-_.~')}={quote(v[0], safe='-_.~')}" for k, v in sorted(query.items()) + ) + canonical = "\n".join( + (method, parts.path, canonical_query, f"host:{parts.netloc}\n", "host", "UNSIGNED-PAYLOAD") + ) + credential = query["X-Amz-Credential"][0].split("/") + scope = "/".join(credential[1:]) + signing = ("AWS4" + secret).encode() + for component in credential[1:]: + signing = hmac.new(signing, component.encode(), hashlib.sha256).digest() + string_to_sign = "\n".join( + ( + "AWS4-HMAC-SHA256", + query["X-Amz-Date"][0], + scope, + hashlib.sha256(canonical.encode()).hexdigest(), + ) + ) + return hmac.new(signing, string_to_sign.encode(), hashlib.sha256).hexdigest() + + +@pytest.mark.parametrize("variant", ["analysis", "original"]) +def test_real_sigv4_binds_method_host_key_version_and_ttl_without_network(variant): + storage = S3Storage(settings()) + source = reference(variant, object_version="version/+?=&:v1") + instant = datetime(2026, 10, 2, 1, 2, 3) + try: + with ( + patch("botocore.auth.get_current_datetime", return_value=instant), + patch( + "botocore.httpsession.URLLib3Session.send", side_effect=AssertionError("network") + ), + ): + grant = storage.presign_image_get(source) + parts, query = urlsplit(grant.url), parse_qs(urlsplit(grant.url).query) + assert parts.scheme == "https" and parts.netloc == "labsafe.test" + assert parts.path == f"/{BUCKET}/{source.key}" + assert query["versionId"] == [source.object_version] + assert query["X-Amz-Expires"] == ["60"] and query["X-Amz-SignedHeaders"] == ["host"] + assert grant.expires_at == instant + timedelta(seconds=60) + signature = query["X-Amz-Signature"][0] + assert signature_for(grant.url, storage.settings.secret_key) == signature + assert signature_for(grant.url, storage.settings.secret_key, method="PUT") != signature + assert signature_for(grant.url, storage.settings.secret_key, version="latest") != signature + assert "Signature" not in repr(grant) + finally: + storage.close() + + +@pytest.mark.parametrize( + "values", + [ + {"key": "staging/x"}, + {"object_version": ""}, + {"object_version": "null"}, + {"object_version": "x" * 201}, + {"object_version": "bad\x00"}, + {"mime_type": "image/jpeg"}, + {"tenant_id": "../x"}, + {"sha256": "INVALID"}, + ], +) +def test_signer_rejects_noncanonical_reference_without_sdk_call(values): + storage = S3Storage(settings()) + try: + with patch.object(storage.signing, "generate_presigned_url") as sign: + with pytest.raises(ValueError): + storage.presign_image_get(reference(**values)) + sign.assert_not_called() + finally: + storage.close() + + +@pytest.mark.parametrize("mutation", ["host", "version", "ttl", "extra", "duplicate", "signature"]) +def test_signer_rejects_malformed_or_misdirected_generated_url(mutation): + storage = S3Storage(settings()) + try: + source = reference() + url = urlsplit(storage.presign_image_get(source).url) + query = parse_qs(url.query) + if mutation == "host": + url = url._replace(netloc="untrusted.test") + elif mutation == "version": + query["versionId"] = ["latest"] + elif mutation == "ttl": + query["X-Amz-Expires"] = ["600"] + elif mutation == "extra": + query["response-content-type"] = ["image/svg+xml"] + elif mutation == "duplicate": + query["versionId"].append("latest") + else: + query["X-Amz-Signature"] = ["private-error"] + malformed = urlunsplit(url._replace(query=urlencode(query, doseq=True))) + with patch.object(storage.signing, "generate_presigned_url", return_value=malformed): + with pytest.raises(ServiceError) as caught: + storage.presign_image_get(source) + assert caught.value.code == "DEPENDENCY_UNAVAILABLE" + assert "private-error" not in str(caught.value) + finally: + storage.close() + + +def test_signer_failure_sanitized(): + storage = S3Storage(settings()) + try: + with patch.object( + storage.signing, + "generate_presigned_url", + side_effect=EndpointConnectionError(endpoint_url="private-secret"), + ): + with pytest.raises(ServiceError) as caught: + storage.presign_image_get(reference()) + assert caught.value.code == "DEPENDENCY_UNAVAILABLE" + assert "private-secret" not in str(caught.value) + finally: + storage.close() + + +def test_download_rate_uses_shared_read_quota_without_fallback_or_write_charge(): + limits = RateLimits(Mock()) + with patch.object(limits, "_charge") as charge: + limits.download_grant(ACTOR) + charge.assert_called_once_with("user", f"{TENANT}:{ID}", 120, 60) diff --git a/tests/business/test_image_validation.py b/tests/business/test_image_validation.py new file mode 100644 index 0000000..92a2f55 --- /dev/null +++ b/tests/business/test_image_validation.py @@ -0,0 +1,387 @@ +"""Real Pillow and real SDK stubs; no live MinIO/IAM acceptance claim.""" + +import hashlib +import multiprocessing +import time +from io import BytesIO +from threading import Event, Timer +from unittest.mock import Mock, patch +from uuid import uuid4 + +import pytest +from botocore.stub import Stubber +from PIL import Image, PngImagePlugin + +from apps.worker.image_validation import BoundedImagePrepare +from packages.application.image_validation import dimensions, normalize, prepare_image +from packages.domain.image_validation import RejectedImage, ValidatedImage, image_keys +from packages.domain.job_execution import ImageInput, LeaseLost +from packages.domain.security import ServiceError +from packages.storage.s3 import BUCKET, S3Settings, S3Storage +from tests.business.test_s3_storage import settings + + +def encoded(format="PNG", mode="RGB", size=(7, 5), **options): + with Image.new(mode, size, 1) as source: + output = BytesIO() + source.save(output, format=format, **options) + return output.getvalue() + + +def image_input(payload, mime="image/png"): + tenant, upload = str(uuid4()), str(uuid4()) + return tenant, ImageInput( + str(uuid4()), + upload, + str(uuid4()), + "inspection_item", + str(uuid4()), + f"staging/{tenant}/{upload}", + "pinned", + hashlib.sha256(payload).hexdigest(), + len(payload), + mime, + 1, + 2, + ) + + +@pytest.mark.parametrize( + "format,mime", [("JPEG", "image/jpeg"), ("PNG", "image/png"), ("WEBP", "image/webp")] +) +def test_real_formats_normalize_without_resize(format, mime): + payload, width, height = normalize(encoded(format), mime) + with Image.open(BytesIO(payload)) as result: + assert (result.format, result.mode, result.size) == ("PNG", "RGB", (7, 5)) + assert not result.info and not result.getexif() + result.load() + assert (width, height) == (7, 5) + + +@pytest.mark.parametrize("mode", ["RGBA", "P", "L", "LA", "I;16"]) +def test_png_modes_become_rgb(mode): + output, _, _ = normalize(encoded(mode=mode), "image/png") + with Image.open(BytesIO(output)) as result: + assert result.mode == "RGB" + + +def test_exif_orientation_once_and_strip_gps_icc_text(): + exif = Image.Exif() + exif[274], exif[270] = 6, "synthetic-private-description" + pnginfo = PngImagePlugin.PngInfo() + pnginfo.add_text("GPS", "synthetic-position") + source = encoded(exif=exif, pnginfo=pnginfo, icc_profile=b"synthetic-profile") + output, width, height = normalize(source, "image/png") + assert (width, height) == (5, 7) + with Image.open(BytesIO(output)) as image: + assert image.info == {} and len(image.getexif()) == 0 + repeated, w2, h2 = normalize(output, "image/png") + assert (w2, h2) == (5, 7) and repeated == output + + +@pytest.mark.parametrize( + "payload,mime,code", + [ + (b"", "image/png", "IMAGE_INVALID"), + (b"not an image", "image/png", "IMAGE_INVALID"), + (encoded(), "image/jpeg", "UNSUPPORTED_MEDIA_TYPE"), + (encoded()[:40], "image/png", "IMAGE_INVALID"), + (encoded("BMP"), "image/png", "UNSUPPORTED_MEDIA_TYPE"), + ], +) +def test_invalid_content_rejected(payload, mime, code): + with pytest.raises(ServiceError) as caught: + normalize(payload, mime) + assert caught.value.code == code + + +@pytest.mark.parametrize("format,mime", [("PNG", "image/png"), ("WEBP", "image/webp")]) +def test_animation_rejected(format, mime): + with Image.new("RGB", (3, 3), "red") as first, Image.new("RGB", (3, 3), "blue") as second: + stream = BytesIO() + first.save(stream, format=format, save_all=True, append_images=[second], duration=100) + with pytest.raises(ServiceError) as caught: + normalize(stream.getvalue(), mime) + assert caught.value.code == "IMAGE_INVALID" + + +@pytest.mark.parametrize( + "size,allowed", + [ + ((10000, 4000), True), + ((10000, 4001), False), + ((10001, 1), False), + ((1, 10001), False), + ((0, 1), False), + ], +) +def test_dimension_boundaries(size, allowed): + if allowed: + dimensions(*size) + else: + with pytest.raises(ServiceError, match="dimensions"): + dimensions(*size) + + +def test_oversized_real_image_header_rejected_before_full_decode(): + with pytest.raises(ServiceError) as caught: + normalize(encoded(size=(10001, 1)), "image/png") + assert caught.value.code == "IMAGE_TOO_LARGE" + + +def test_prepare_sdk_exact_get_copy_and_png_put_versions(): + original = encoded() + tenant, input = image_input(original) + analysis, w, h = normalize(original, input.mime_type) + sha = hashlib.sha256(analysis).hexdigest() + original_key, analysis_key = image_keys(tenant, input, sha) + storage = S3Storage(settings()) + body = BytesIO(original) + try: + with Stubber(storage.internal) as stub: + stub.add_response( + "get_object", + { + "Body": body, + "VersionId": "pinned", + "ContentLength": len(original), + "ContentType": "image/png", + }, + {"Bucket": BUCKET, "Key": input.key, "VersionId": "pinned"}, + ) + stub.add_client_error( + "head_object", "NoSuchKey", expected_params={"Bucket": BUCKET, "Key": original_key} + ) + stub.add_response( + "copy_object", + {"VersionId": "original-v1"}, + { + "Bucket": BUCKET, + "Key": original_key, + "CopySource": {"Bucket": BUCKET, "Key": input.key, "VersionId": "pinned"}, + "MetadataDirective": "REPLACE", + "ContentType": "image/png", + }, + ) + stub.add_client_error( + "head_object", "NoSuchKey", expected_params={"Bucket": BUCKET, "Key": analysis_key} + ) + stub.add_response( + "put_object", + {"VersionId": "analysis-v1"}, + { + "Bucket": BUCKET, + "Key": analysis_key, + "Body": analysis, + "ContentType": "image/png", + }, + ) + assert prepare_image(storage, tenant, input) == ValidatedImage( + original_key, "original-v1", analysis_key, "analysis-v1", sha, w, h + ) + stub.assert_no_pending_responses() + finally: + storage.close() + assert body.closed + + +@pytest.mark.parametrize( + "code", + [ + "IMAGE_INVALID", + "HASH_MISMATCH", + "OBJECT_NOT_FOUND", + "IMAGE_TOO_LARGE", + "UNSUPPORTED_MEDIA_TYPE", + ], +) +def test_content_error_never_writes_objects(code): + storage = Mock() + storage.read_staging.side_effect = ServiceError(code, 422, "safe") + tenant, input = image_input(encoded()) + assert prepare_image(storage, tenant, input) == RejectedImage(code) + storage.preserve_original.assert_not_called() + storage.put_analysis.assert_not_called() + + +def test_decode_rejects_after_verified_read_before_writes(): + storage = Mock() + storage.read_staging.return_value = b"not an image" + tenant, input = image_input(b"not an image") + assert prepare_image(storage, tenant, input) == RejectedImage("IMAGE_INVALID") + storage.preserve_original.assert_not_called() + + +def test_storage_unavailable_after_original_write_never_returns_ready(): + storage = Mock() + storage.read_staging.return_value = encoded() + storage.preserve_original.return_value = "orphan-version" + storage.put_analysis.side_effect = ServiceError("DEPENDENCY_UNAVAILABLE", 503, "safe") + tenant, input = image_input(encoded()) + with pytest.raises(ServiceError) as caught: + prepare_image(storage, tenant, input) + assert caught.value.code == "DEPENDENCY_UNAVAILABLE" + storage.delete_object.assert_not_called() + + +@pytest.mark.parametrize("version", [None, "", "null", "x" * 201, "with space"]) +def test_written_objects_require_precise_version(version): + with pytest.raises(ServiceError): + S3Storage.result_version({"VersionId": version}) + + +@pytest.mark.parametrize("variant", ["original", "analysis"]) +@pytest.mark.parametrize("conflict", [False, True]) +def test_existing_destination_reused_only_after_actual_hash(variant, conflict): + payload = encoded() + tenant, input = image_input(payload) + sha = hashlib.sha256(payload).hexdigest() + original, analysis = image_keys(tenant, input, sha) + key = original if variant == "original" else analysis + body = BytesIO(b"x" * len(payload) if conflict else payload) + storage = S3Storage(settings()) + try: + with Stubber(storage.internal) as stub: + stub.add_response( + "head_object", + { + "VersionId": "existing-v1", + "ContentLength": len(payload), + "ContentType": "image/png", + "Metadata": {"sha256": sha}, + }, + {"Bucket": BUCKET, "Key": key}, + ) + stub.add_response( + "get_object", + { + "Body": body, + "VersionId": "existing-v1", + "ContentLength": len(payload), + "ContentType": "image/png", + }, + {"Bucket": BUCKET, "Key": key, "VersionId": "existing-v1"}, + ) + + def write(): + if variant == "original": + from packages.storage.s3 import ObjectVersion + + return storage.preserve_original( + ObjectVersion(input.key, "pinned", len(payload), "image/png"), key, sha + ) + return storage.put_analysis(key, payload) + + if conflict: + with pytest.raises(ServiceError) as caught: + write() + assert caught.value.code == "INTERNAL_ERROR" + else: + assert write() == "existing-v1" + stub.assert_no_pending_responses() # No COPY/PUT on reuse or conflict. + finally: + storage.close() + assert body.closed + + +@pytest.mark.parametrize("provider", ["AccessDenied", "SlowDown", "NoSuchVersion"]) +def test_existing_destination_dependency_failure_never_overwrites(provider): + payload = encoded() + storage = S3Storage(settings()) + try: + with Stubber(storage.internal) as stub: + if provider == "NoSuchVersion": + stub.add_response( + "head_object", + { + "VersionId": "existing-v1", + "ContentLength": len(payload), + "ContentType": "image/png", + }, + {"Bucket": BUCKET, "Key": "controlled-analysis-key"}, + ) + stub.add_client_error("get_object", provider) + else: + stub.add_client_error("head_object", provider) + with pytest.raises(ServiceError) as caught: + storage.put_analysis("controlled-analysis-key", payload) + assert caught.value.code == "DEPENDENCY_UNAVAILABLE" + stub.assert_no_pending_responses() + finally: + storage.close() + + +def stalled_prepare(*args): + time.sleep(60) + + +def synthetic_storage_decode_child(channel, _settings, tenant, input): + # Real spawned process + Pillow, with clearly synthetic storage version IDs. + payload, width, height = normalize(encoded(), input.mime_type) + sha = hashlib.sha256(payload).hexdigest() + original, analysis = image_keys(tenant, input, sha) + channel.send(("result", ValidatedImage(original, "o-v1", analysis, "a-v1", sha, width, height))) + channel.close() + + +def test_spawned_child_returns_real_decode_result(monkeypatch): + monkeypatch.setattr( + "apps.worker.image_validation.prepare_child", synthetic_storage_decode_child + ) + tenant, input = image_input(encoded()) + result = BoundedImagePrepare(settings(), tenant)(input, Event()) + assert (result.width, result.height) == (7, 5) + assert ( + result.analysis_sha256 == hashlib.sha256(normalize(encoded(), "image/png")[0]).hexdigest() + ) + + +@pytest.mark.parametrize("cancel", [False, True]) +def test_real_stalled_child_killed_on_timeout_or_lease_loss(monkeypatch, cancel): + monkeypatch.setattr("apps.worker.image_validation.prepare_child", stalled_prepare) + monkeypatch.setattr("apps.worker.image_validation.PREPARE_SECONDS", 0.4 if not cancel else 5) + tenant, input = image_input(encoded()) + cancelled = Event() + timer = Timer(0.2, cancelled.set) if cancel else None + before = {p.pid for p in multiprocessing.active_children()} + if timer: + timer.start() + try: + with pytest.raises(LeaseLost if cancel else ServiceError) as caught: + BoundedImagePrepare(settings(), tenant)(input, cancelled) + if not cancel: + assert caught.value.code == "STAGE_TIMEOUT" + finally: + if timer: + timer.cancel() + timer.join() + assert {p.pid for p in multiprocessing.active_children()} == before + + +def test_worker_secrets_have_no_api_fallback(monkeypatch): + monkeypatch.delenv("S3_WORKER_ACCESS_KEY_FILE", raising=False) + monkeypatch.delenv("S3_WORKER_SECRET_KEY_FILE", raising=False) + with pytest.raises(ValueError): + S3Settings.from_environment("https://labsafe.test", worker=True) + + +@pytest.mark.parametrize("enabled", [False, True, False]) +def test_consumer_opt_in_and_sanitized_errors(monkeypatch, enabled, caplog): + from apps.worker.app.main import create_celery_app + + monkeypatch.setenv("APP_ENV", "test") + monkeypatch.setenv("WORKER_IMAGE_VALIDATION_ENABLED", "1" if enabled else "0") + monkeypatch.setenv("PUBLIC_ORIGIN", "https://labsafe.test") + with patch.object(S3Settings, "from_environment", return_value=settings()): + app = create_celery_app() + try: + assert ("labsafe.tasks.dispatch" in app.tasks) == enabled + if enabled: + with patch( + "apps.worker.image_validation.consume_image", side_effect=RuntimeError("secret") + ): + app.tasks["labsafe.tasks.dispatch"].run({"untrusted": "private"}) + assert "secret" not in caplog.text and "private" not in caplog.text + assert "image_dispatch_failed" in caplog.text + finally: + app.close() diff --git a/tests/business/test_inference_execution.py b/tests/business/test_inference_execution.py new file mode 100644 index 0000000..36497d0 --- /dev/null +++ b/tests/business/test_inference_execution.py @@ -0,0 +1,114 @@ +from datetime import datetime, timezone +from uuid import uuid4 + +import pytest + +from packages.domain.inference_execution import ( + InferenceImage, + InferenceInput, + InferenceInvalidResult, + InferenceLease, + request_payload, + validate_result, +) + + +def lease(): + tenant, lab, item, run, image = (str(uuid4()) for _ in range(5)) + value = InferenceInput( + tenant_id=tenant, + laboratory_id=lab, + item_id=item, + run_id=run, + submission_revision=2, + model_bundle_id=str(uuid4()), + model_checksum="a" * 64, + dictionary_version_id=str(uuid4()), + dictionary_sha256="b" * 64, + pipeline_version="vision-v1", + device_profile="cpu", + images=( + InferenceImage( + image_id=image, + object_key=f"tenant/{tenant}/lab/{lab}/analysis/{image}/{'c' * 64}.png", + sha256="c" * 64, + mime_type="image/png", + role="overview", + location_id=str(uuid4()), + parent_image_id=None, + ), + ), + ) + return InferenceLease(tenant, str(uuid4()), str(uuid4()), str(uuid4()), 3, 0, 1, value) + + +def result_for(value, outcome="needs_retake"): + image = value.input.images[0] + return { + "run_id": value.input.run_id, + "attempt_id": value.attempt_id, + "fencing_token": value.token, + "tenant_id": value.input.tenant_id, + "request_hash": value.input.request_hash, + "input_hashes": [{"image_id": image.image_id, "sha256": image.sha256}], + "model_bundle_id": value.input.model_bundle_id, + "model_checksum": value.input.model_checksum, + "dictionary_version_id": value.input.dictionary_version_id, + "dictionary_sha256": value.input.dictionary_sha256, + "pipeline_version": "vision-v1", + "outcome": outcome, + "quality": [ + { + "image_id": image.image_id, + "status": "needs_retake" if outcome == "needs_retake" else "pass", + "reasons": ["blur"] if outcome == "needs_retake" else [], + "blur_score": 0 if outcome == "needs_retake" else 100, + "brightness": 0.5, + "glare_ratio": 0, + } + ], + "detections": [], + "crops": [], + "ocr_fields": [], + "entities": [], + "relations": [], + "timing_ms": { + "download_ms": 0, + "quality_ms": 0, + "detection_ms": 0, + "ocr_ms": 0, + "normalization_ms": 0, + "total_ms": 0, + }, + } + + +def test_request_payload_contains_fixed_input_and_fence(): + value = lease() + payload = request_payload(value, datetime(2026, 10, 4, tzinfo=timezone.utc)) + assert payload["attempt_id"] == value.attempt_id + assert payload["fencing_token"] == value.token + assert payload["request_hash"] == value.input.request_hash + assert payload["image_refs"][0]["object_key"] == value.input.images[0].object_key + + +def test_result_accepts_exact_echo_and_input_hashes(): + value = lease() + validate_result(result_for(value), value) + + +@pytest.mark.parametrize("field", ["run_id", "attempt_id", "fencing_token", "request_hash"]) +def test_result_rejects_stale_fence_or_identity(field): + value = lease() + result = result_for(value) + result[field] = str(uuid4()) if field != "fencing_token" else value.token + 1 + with pytest.raises(InferenceInvalidResult): + validate_result(result, value) + + +def test_result_rejects_different_image_hash(): + value = lease() + result = result_for(value) + result["input_hashes"][0]["sha256"] = "d" * 64 + with pytest.raises(InferenceInvalidResult): + validate_result(result, value) diff --git a/tests/business/test_job_api.py b/tests/business/test_job_api.py new file mode 100644 index 0000000..6354de5 --- /dev/null +++ b/tests/business/test_job_api.py @@ -0,0 +1,256 @@ +"""Job routes and pure replay boundaries, without real DB or object server.""" + +from dataclasses import replace +from unittest.mock import Mock, patch + +import pytest +from botocore.stub import Stubber +from fastapi.testclient import TestClient + +from apps.api.app.main import create_app +from packages.domain.job_replay import replay_body, require_replay +from packages.domain.security import Principal, ServiceError +from packages.storage.s3 import BUCKET, ObjectVersion, S3Storage +from tests.business.test_foundation_boundary import HEADERS, ID, ORIGIN +from tests.business.test_s3_storage import KEY, TENANT, UPLOAD, settings +from tests.helpers import configured + + +def client(service): + with configured(): + app = create_app(identity=Mock(), public_origin=ORIGIN) + app.state.jobs = service + return TestClient(app, base_url=ORIGIN) + + +def body(): + return {"expected_version": 1, "reason": "Retry after storage recovery"} + + +def test_job_routes_normalize_ids_and_preserve_202(): + service = Mock() + service.read.return_value = {"data": {}, "request_id": ID} + service.replay.return_value = (202, {"data": {"state": "ready"}, "request_id": ID}) + with client(service) as http: + assert http.get(f"/api/v1/jobs/{ID.upper()}").status_code == 200 + assert service.read.call_args.kwargs["job_id"] == ID + assert ( + http.get( + f"/api/v1/dead-letters?page=2&page_size=3&laboratory_id={ID.upper()}" + ).status_code + == 200 + ) + assert service.read.call_args.kwargs == {"page": 2, "page_size": 3, "laboratory_id": ID} + response = http.post( + f"/api/v1/dead-letters/{ID.upper()}/replay", json=body(), headers=HEADERS + ) + assert response.status_code == 202 and response.headers["cache-control"] == "no-store" + assert service.replay.call_args.args[3:5] == (ID, body()) + + +@pytest.mark.parametrize( + "values", + [ + {"expected_version": True}, + {"expected_version": "1"}, + {"expected_version": 0}, + {"expected_version": 2147483648}, + {"reason": " "}, + {"reason": ""}, + {"reason": "x" * 2001}, + {"tenant_id": ID}, + {"generation": 1}, + {"object_version": "latest"}, + ], +) +def test_replay_rejects_non_contract_body(values): + service = Mock() + with client(service) as http: + assert ( + http.post( + f"/api/v1/dead-letters/{ID}/replay", json={**body(), **values}, headers=HEADERS + ).status_code + == 422 + ) + service.replay.assert_not_called() + + +@pytest.mark.parametrize( + "path", + [ + f"/api/v1/jobs/{ID}?state=failed", + "/api/v1/dead-letters?state=failed", + "/api/v1/dead-letters?page=0", + "/api/v1/dead-letters?page_size=101", + "/api/v1/dead-letters?page=1&page=2", + ], +) +def test_job_queries_reject_extra_duplicate_or_invalid_parameters(path): + service = Mock() + with client(service) as http: + assert http.get(path).status_code == 422 + service.read.assert_not_called() + + +def test_replay_keeps_security_headers_origin_json_and_no_query(): + service = Mock() + path = f"/api/v1/dead-letters/{ID}/replay" + with client(service) as http: + assert http.post(path, json=body()).status_code == 403 + assert http.post(path, content="text", headers=HEADERS).status_code == 415 + assert http.post(path + "?version=latest", json=body(), headers=HEADERS).status_code == 422 + for missing in ("X-CSRF-Token", "Idempotency-Key"): + headers = {k: v for k, v in HEADERS.items() if k.lower() != missing.lower()} + assert http.post(path, json=body(), headers=headers).status_code == 422 + assert ( + http.post( + path, json=body(), headers=[*HEADERS.items(), ("Idempotency-Key", "duplicate")] + ).status_code + == 422 + ) + service.replay.assert_not_called() + + +def test_disabled_job_service_fails_closed(): + with client(None) as http: + assert http.get(f"/api/v1/jobs/{ID}").status_code == 503 + assert http.get("/api/v1/dead-letters").status_code == 503 + assert ( + http.post(f"/api/v1/dead-letters/{ID}/replay", json=body(), headers=HEADERS).status_code + == 503 + ) + + +ACTOR = Principal(ID, TENANT, "synthetic", (("safety_admin", "tenant", None),)) +TASK = { + "tenant_id": TENANT, + "task_type": "validate_image", + "version": 1, + "state": "failed", + "replay_generation": 0, + "dispatch_sequence": 1, +} + + +@pytest.mark.parametrize( + "state,allowed", + [ + ("failed", True), + ("dead_letter", True), + ("succeeded", False), + ("leased", False), + ("ready", False), + ("retry_wait", False), + ], +) +def test_replay_only_technical_terminal_task(state, allowed): + if allowed: + require_replay(ACTOR, {**TASK, "state": state}, object(), 1) + else: + with pytest.raises(ServiceError) as caught: + require_replay(ACTOR, {**TASK, "state": state}, object(), 1) + assert caught.value.code == "STATE_CONFLICT" + + +@pytest.mark.parametrize( + "kind", ["no_admin", "tenant", "unsupported", "missing", "source", "version", "counter"] +) +def test_replay_guard_rejects_unauthorized_stale_or_invalid_source(kind): + actor, task, source, version = ACTOR, TASK, object(), 1 + if kind == "no_admin": + actor = replace(ACTOR, roles=(("viewer", "laboratory", ID),)) + elif kind == "tenant": + task = {**TASK, "tenant_id": ID} + elif kind == "unsupported": + task = {**TASK, "task_type": "inference_pipeline"} + elif kind == "missing": + task = None + elif kind == "source": + source = None + elif kind == "version": + version = 2 + else: + task = {**TASK, "dispatch_sequence": 2147483647} + with pytest.raises(ServiceError): + require_replay(actor, task, source, version) + + +@pytest.mark.parametrize( + "value", + [ + None, + {}, + {"expected_version": 1, "reason": " "}, + {"expected_version": True, "reason": "x"}, + {"expected_version": 1, "reason": "x", "tenant": ID}, + ], +) +def test_internal_replay_body_is_strict(value): + with pytest.raises(ServiceError): + replay_body(value) + + +@pytest.mark.parametrize( + "change", + [ + {}, + {"VersionId": "new-v2"}, + {"ContentLength": 11}, + {"ContentType": "image/jpeg"}, + {"DeleteMarker": True}, + ], +) +def test_replay_sdk_head_uses_exact_version_and_checks_metadata(change): + storage = S3Storage(settings()) + pinned = ObjectVersion(KEY, "v1", 10, "image/png") + try: + with Stubber(storage.internal) as stub: + stub.add_response( + "head_object", + {"VersionId": "v1", "ContentLength": 10, "ContentType": "image/png", **change}, + {"Bucket": BUCKET, "Key": KEY, "VersionId": "v1"}, + ) + if change: + with pytest.raises(ServiceError) as caught: + storage.inspect_pinned_staging(TENANT, UPLOAD, pinned) + assert caught.value.code == "STATE_CONFLICT" + else: + assert storage.inspect_pinned_staging(TENANT, UPLOAD, pinned) == pinned + stub.assert_no_pending_responses() + finally: + storage.close() + + +@pytest.mark.parametrize( + "provider,code", + [("NoSuchVersion", "OBJECT_NOT_FOUND"), ("AccessDenied", "DEPENDENCY_UNAVAILABLE")], +) +def test_replay_head_missing_version_or_storage_failure_no_fallback(provider, code): + storage = S3Storage(settings()) + try: + with Stubber(storage.internal) as stub: + stub.add_client_error( + "head_object", + provider, + expected_params={"Bucket": BUCKET, "Key": KEY, "VersionId": "v1"}, + ) + with pytest.raises(ServiceError) as caught: + storage.inspect_pinned_staging( + TENANT, UPLOAD, ObjectVersion(KEY, "v1", 10, "image/png") + ) + assert caught.value.code == code + finally: + storage.close() + + +def test_foreign_pinned_reference_never_accesses_storage(): + storage = S3Storage(settings()) + try: + with patch.object(storage.internal, "head_object") as head: + with pytest.raises(ValueError): + storage.inspect_pinned_staging( + TENANT, UPLOAD, ObjectVersion("foreign/key", "v1", 10, "image/png") + ) + head.assert_not_called() + finally: + storage.close() diff --git a/tests/business/test_job_execution.py b/tests/business/test_job_execution.py new file mode 100644 index 0000000..3bbfd89 --- /dev/null +++ b/tests/business/test_job_execution.py @@ -0,0 +1,88 @@ +import json +from pathlib import Path +from threading import Event +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from packages.application.job_execution import ImageExecution, LeaseHeartbeat +from packages.domain.job_execution import TECHNICAL_ERRORS, LeaseLost, retry_delay + + +@pytest.mark.parametrize("attempt,seconds", [(1, 5), (2, 30), (3, 120)]) +@pytest.mark.parametrize("jitter", [0, 0.1, 0.2]) +def test_retry_delay_matches_policy(attempt, seconds, jitter): + actual = retry_delay(attempt, uniform=lambda low, high: jitter) + assert actual.total_seconds() == seconds * (1 + jitter) + + +@pytest.mark.parametrize("attempt", [0, 4, True, -1, "1"]) +def test_retry_delay_refuses_invalid_attempt(attempt): + with pytest.raises(ValueError): + retry_delay(attempt) + + +def test_failure_classification_matches_authoritative_contract(): + contract = json.loads( + (Path(__file__).resolve().parents[2] / "contracts/error-codes.json").read_text() + ) + assert all( + contract[code]["retryable"] == retryable for code, retryable in TECHNICAL_ERRORS.items() + ) + + +@pytest.mark.parametrize("failure", [False, RuntimeError("secret database URL")]) +def test_heartbeat_failure_latches_cancels_and_prevents_success_exit(failure, caplog): + service, cancel = Mock(), Mock() + if isinstance(failure, Exception): + service.heartbeat.side_effect = failure + else: + service.heartbeat.return_value = failure + keeper = LeaseHeartbeat(service, SimpleNamespace(task_id="test-task"), cancel) + keeper.interval = 0.01 + with pytest.raises(LeaseLost): + with keeper: + assert keeper.lost.wait(2) + cancel.assert_called_once() + assert "secret database" not in caplog.text + assert not keeper.thread.is_alive() + + +def test_heartbeat_clean_exit_stops_thread(): + service = Mock() + keeper = LeaseHeartbeat(service, SimpleNamespace(task_id="test-task"), Mock()) + with keeper: + pass + assert not keeper.thread.is_alive() + service.heartbeat.assert_not_called() + + +def test_execution_is_not_released_for_production(monkeypatch): + monkeypatch.setenv("APP_ENV", "production") + with pytest.raises(RuntimeError): + ImageExecution(Mock()) + + +@pytest.mark.parametrize("cancel_raises", [False, True]) +def test_execute_never_commits_after_transient_heartbeat_failure(monkeypatch, cancel_raises): + monkeypatch.setenv("APP_ENV", "test") + monkeypatch.setattr(LeaseHeartbeat, "interval", 0.01) + execution = ImageExecution(Mock()) + execution.claim = Mock(return_value=SimpleNamespace(task_id="test-task")) + execution.heartbeat = Mock(side_effect=[False, True]) + execution.commit = Mock() + execution.fail = Mock() + + def external_work(input, cancelled): + assert isinstance(cancelled, Event) + assert cancelled.wait(2) + if cancel_raises: + raise RuntimeError("external work cancelled") + return "discard-me" + + execution.claim.return_value.input = None + with pytest.raises(LeaseLost): + execution.execute({}, external_work, Mock()) + execution.commit.assert_not_called() + execution.fail.assert_not_called() diff --git a/tests/business/test_review_boundary.py b/tests/business/test_review_boundary.py new file mode 100644 index 0000000..2eab290 --- /dev/null +++ b/tests/business/test_review_boundary.py @@ -0,0 +1,136 @@ +"""HTTP contract tests for facts and finding review endpoints.""" + +from unittest.mock import Mock +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient + +from apps.api.app.main import create_app +from packages.persistence.review import _public_facts +from tests.business.test_foundation_boundary import HEADERS, ID +from tests.business.test_identity_boundary import ORIGIN +from tests.helpers import configured + + +def client(service): + with configured(): + app = create_app(identity=Mock(), public_origin=ORIGIN) + app.state.review = service + return TestClient(app, base_url=ORIGIN) + + +FACTS = { + "expected_version": 1, + "entities": [], + "relations": [], + "dates": [], + "reason": "Corrected from the inspection label", +} +DECISION = {"expected_version": 1, "reason": "Reviewed against the image"} + + +@pytest.mark.parametrize( + "path,body,operation", + [ + (f"/inspection-items/{ID}/facts", FACTS, "facts"), + (f"/findings/{ID}/confirm", DECISION, "confirm"), + (f"/findings/{ID}/reject", DECISION, "reject"), + ( + f"/findings/{ID}/cannot-determine", + {**DECISION, "reason_code": "blur"}, + "cannot-determine", + ), + ], +) +def test_review_write_routes_preserve_status_and_payload(path, body, operation): + service = Mock() + service.write.return_value = ( + 202 if operation == "facts" else 200, + {"data": {}, "request_id": str(uuid4())}, + ) + with client(service) as http: + response = http.post("/api/v1" + path, json=body, headers=HEADERS) + assert response.status_code == service.write.return_value[0], response.text + assert response.headers["cache-control"] == "no-store" + assert service.write.call_args.args[0] == operation + assert service.write.call_args.args[4] == ID + assert service.write.call_args.args[5] == body + + +@pytest.mark.parametrize( + "path,kind,resource_id", + [ + (f"/inspection-items/{ID}/facts", "facts", ID), + (f"/findings/{ID}", "finding", ID), + ("/findings", "findings", None), + ], +) +def test_review_read_routes_and_query_allowlist(path, kind, resource_id): + service = Mock() + service.read.return_value = {"data": {}, "request_id": str(uuid4())} + with client(service) as http: + assert http.get("/api/v1" + path).status_code == 200 + assert http.get("/api/v1" + path + "?unexpected=1").status_code == 422 + if kind == "findings": + assert ( + http.get( + "/api/v1/findings?page=2&page_size=3¤t_only=false" + "&status=needs_review&severity=critical&laboratory_id=" + ID + ).status_code + == 200 + ) + assert http.get("/api/v1/findings?current_only=1").status_code == 422 + assert http.get("/api/v1/findings?status=bogus").status_code == 422 + else: + assert http.get("/api/v1" + path + "?page=2").status_code == 422 + assert service.read.call_args.args[0] == kind + assert service.read.call_args.kwargs["resource_id"] == resource_id + + +@pytest.mark.parametrize( + "body", + [ + {**FACTS, "tenant_id": ID}, + {**FACTS, "expected_version": True}, + {**FACTS, "entities": [{"detection_id": ID}]}, + {**FACTS, "relations": [{"source_detection_id": ID, "target_detection_id": ID}]}, + {**FACTS, "dates": [{"detection_id": ID, "kind": "expiry", "value": "bad"}]}, + ], +) +def test_facts_edit_rejects_non_contract_body(body): + service = Mock() + with client(service) as http: + response = http.post(f"/api/v1/inspection-items/{ID}/facts", json=body, headers=HEADERS) + assert response.status_code == 422, response.text + service.write.assert_not_called() + + +def test_review_routes_require_write_headers_and_fail_closed_without_service(): + with client(None) as http: + assert http.get(f"/api/v1/findings/{ID}").status_code == 503 + assert http.post(f"/api/v1/findings/{ID}/reject", json=DECISION).status_code == 403 + assert ( + http.post(f"/api/v1/findings/{ID}/reject", content="x", headers=HEADERS).status_code + == 415 + ) + + +def test_model_facts_are_projected_to_public_fact_revision_shape(): + image_id, detection_id, entity_id = (str(uuid4()) for _ in range(3)) + entity = { + "image_id": image_id, + "detection_id": detection_id, + "resolution": "resolved", + "candidates": [{"entity_id": entity_id}], + } + projected = _public_facts("entities", [entity]) + assert projected == [ + { + "detection_id": detection_id, + "entity_id": entity_id, + "resolution": "resolved", + "source": "model", + "evidence": [{"image_id": image_id, "detection_id": detection_id, "crop_id": None}], + } + ] diff --git a/tests/business/test_rule_evaluator.py b/tests/business/test_rule_evaluator.py new file mode 100644 index 0000000..77cc04d --- /dev/null +++ b/tests/business/test_rule_evaluator.py @@ -0,0 +1,30 @@ +import json +from pathlib import Path + +import pytest + +from packages.rules.evaluator import RuleDefinitionError, evaluate_bundle, evaluate_rule + +CASES = json.loads( + ( + Path(__file__).resolve().parents[2] / "contracts/examples/synthetic-rule-cases.json" + ).read_text(encoding="utf-8") +) + + +@pytest.mark.parametrize("case", CASES["cases"]) +def test_synthetic_cases_preserve_three_valued_truth(case): + rule = next(rule for rule in CASES["bundle"]["rules"] if rule["rule_id"] == case["rule_id"]) + actual, _ = evaluate_rule(rule, case["facts"], case["reference_date"]) + assert actual is case["expected"] + + +def test_bundle_evaluation_is_priority_then_rule_id(): + result = evaluate_bundle(CASES["bundle"], CASES["cases"][0]["facts"], "2026-09-27") + assert [item["rule_id"] for item in result] == ["SYNTHETIC-DATE", "SYNTHETIC-PAIR"] + + +def test_duplicate_rule_ids_are_rejected(): + bundle = dict(CASES["bundle"], rules=CASES["bundle"]["rules"] * 2) + with pytest.raises(RuleDefinitionError): + evaluate_bundle(bundle, {}, "2026-09-27") diff --git a/tests/business/test_rules_boundary.py b/tests/business/test_rules_boundary.py new file mode 100644 index 0000000..d90917b --- /dev/null +++ b/tests/business/test_rules_boundary.py @@ -0,0 +1,131 @@ +"""HTTP adapter checks for the rule configuration lifecycle.""" + +from unittest.mock import Mock +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient + +from apps.api.app.main import create_app +from tests.business.test_identity_boundary import ORIGIN +from tests.helpers import configured + +RULE_SET_ID = str(uuid4()) +RULE_VERSION_ID = str(uuid4()) +HEADERS = { + "Origin": ORIGIN, + "Idempotency-Key": "rules-test-key", + "X-CSRF-Token": "s" * 43, +} +RULE_BUNDLE = { + "rule_set_id": RULE_SET_ID, + "version_label": "synthetic-v1", + "rules": [], + "evaluator_version": "rules-dnf-v1", +} + + +def client(service): + with configured(): + app = create_app(identity=Mock(), public_origin=ORIGIN) + app.state.rules = service + return TestClient(app, base_url=ORIGIN) + + +@pytest.mark.parametrize( + "method,path,body,operation", + [ + ("post", "/rule-sets", {"name": "Rules", "description": "Synthetic"}, "create"), + ("post", "/rule-versions", {"bundle": RULE_BUNDLE, "reason": "Import"}, "import"), + ( + "post", + f"/rule-versions/{RULE_VERSION_ID}/submit-approval", + {"expected_version": 1, "reason": "Submit"}, + "submit", + ), + ( + "post", + f"/rule-versions/{RULE_VERSION_ID}/approve", + {"expected_version": 2, "reason": "Approve"}, + "approve", + ), + ( + "post", + f"/rule-versions/{RULE_VERSION_ID}/publish", + {"expected_version": 3, "reason": "Publish"}, + "publish", + ), + ( + "post", + f"/rule-versions/{RULE_VERSION_ID}/retire", + {"expected_version": 4, "reason": "Retire"}, + "retire", + ), + ( + "post", + "/rule-bundles", + {"rule_version_ids": [RULE_VERSION_ID]}, + "create_bundle", + ), + ], +) +def test_rule_write_contract(method, path, body, operation): + service = Mock() + service.write.return_value = ( + 201 if operation in {"create", "import", "create_bundle"} else 200, + {"data": {}, "request_id": str(uuid4())}, + ) + with client(service) as http: + response = getattr(http, method)("/api/v1" + path, json=body, headers=HEADERS) + assert response.status_code == service.write.return_value[0], response.text + assert response.headers["cache-control"] == "no-store" + assert service.write.call_args.args[0:2] == ( + "rule_set" + if operation == "create" + else "rule_version" + if operation in {"import", "submit", "approve", "publish", "retire"} + else "rule_bundle", + "create" if operation == "create_bundle" else operation, + ) + + +@pytest.mark.parametrize( + "path,kind,resource_id", + [ + ("/rule-sets", "rule_set", None), + ("/rule-versions", "rule_version", None), + (f"/rule-versions/{RULE_VERSION_ID}", "rule_version", RULE_VERSION_ID), + ("/rule-bundles", "rule_bundle", None), + ], +) +def test_rule_read_contract_and_query_allowlist(path, kind, resource_id): + service = Mock() + service.read.return_value = {"data": {}, "request_id": str(uuid4())} + with client(service) as http: + response = http.get("/api/v1" + path) + assert response.status_code == 200, response.text + assert http.get("/api/v1" + path + "?unexpected=1").status_code == 422 + if resource_id is None: + assert http.get("/api/v1" + path + "?page=2&page_size=3").status_code == 200 + assert http.get("/api/v1" + path + "?laboratory_id=bad").status_code == 422 + assert service.read.call_args.args[0] == kind + assert service.read.call_args.kwargs["resource_id"] == resource_id + + +def test_rule_write_rejects_extra_fields_missing_headers_and_unexpected_query(): + service = Mock() + with client(service) as http: + body = {"name": "Rules", "description": "Synthetic"} + assert ( + http.post( + "/api/v1/rule-sets", json={**body, "tenant_id": str(uuid4())}, headers=HEADERS + ).status_code + == 422 + ) + assert http.post("/api/v1/rule-sets", json=body).status_code == 403 + assert ( + http.post("/api/v1/rule-sets?unexpected=1", json=body, headers=HEADERS).status_code + == 422 + ) + assert http.post("/api/v1/rule-sets", content="x", headers=HEADERS).status_code == 415 + service.write.assert_not_called() diff --git a/tests/persistence/test_dispatch_mysql.py b/tests/persistence/test_dispatch_mysql.py new file mode 100644 index 0000000..b0416e6 --- /dev/null +++ b/tests/persistence/test_dispatch_mysql.py @@ -0,0 +1,296 @@ +"""Real MySQL scheduling races and real Redis/Celery transport recovery.""" + +import json +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier +from unittest.mock import Mock +from uuid import uuid4 + +import pytest +from sqlalchemy import event, text + +from apps.worker.app.main import create_celery_app +from apps.worker.dispatch import CeleryDispatchTransport +from packages.application.dispatch import DispatchPublisher, sweep_dispatch +from packages.persistence import dispatch +from packages.persistence.image_jobs import enqueue_image_validation +from tests.persistence.factories import insert + + +@pytest.fixture +def database(database): + # Scheduler connections use RC; API/query tests retain their normal RR engine. + return database.execution_options(isolation_level="READ COMMITTED") + + +@pytest.fixture +def intent(database): + tenant = str(uuid4()) + with database.begin() as connection: + insert(connection, "tenants", id=tenant, timezone="UTC") + task = enqueue_image_validation( + connection, + tenant_id=tenant, + upload_id=str(uuid4()), + image_id=str(uuid4()), + request_id=str(uuid4()), + ) + # Deterministic first candidate without depending on other suite fixtures. + connection.execute( + text("UPDATE outbox_events SET available_at='2000-01-01' WHERE tenant_id=:tenant"), + {"tenant": tenant}, + ) + yield tenant, task + with database.begin() as connection: + connection.execute( + text("DELETE FROM outbox_events WHERE tenant_id=:tenant"), {"tenant": tenant} + ) + connection.execute( + text("DELETE FROM task_runs WHERE tenant_id=:tenant"), {"tenant": tenant} + ) + connection.execute(text("DELETE FROM tenants WHERE id=:tenant"), {"tenant": tenant}) + + +def rows(database, intent): + with database.connect() as connection: + return [ + dict(row) + for row in connection.execute( + text("SELECT * FROM outbox_events WHERE tenant_id=:tenant ORDER BY created_at,id"), + {"tenant": intent[0]}, + ).mappings() + ] + + +def task_row(database, intent): + with database.connect() as connection: + return dict( + connection.execute( + text("SELECT * FROM task_runs WHERE tenant_id=:tenant AND id=:id"), + {"tenant": intent[0], "id": intent[1]}, + ) + .mappings() + .one() + ) + + +def expire(database, intent): + with database.begin() as connection: + connection.execute( + text( + "UPDATE outbox_events SET lease_until=UTC_TIMESTAMP(3)-" + "INTERVAL 1 SECOND WHERE tenant_id=:tenant AND state='leased'" + ), + {"tenant": intent[0]}, + ) + + +def test_publish_after_transaction_and_stable_broker_id(database, intent): + transport = Mock() + + def publish(event_id, message, renew): + current = rows(database, intent)[0] + assert current["state"] == "leased" and current["id"] == event_id + assert message["task_id"] == intent[1] + # Prove broker callback can acquire the row lock from a separate connection. + with database.begin() as connection: + connection.execute( + text("SELECT id FROM outbox_events WHERE id=:id FOR UPDATE NOWAIT"), + {"id": event_id}, + ) + renew() + + transport.publish.side_effect = publish + assert DispatchPublisher(database, transport).publish_batch(1) == 1 + row = rows(database, intent)[0] + assert row["state"] == "published" and row["published_at"] is not None + assert row["lease_owner"] is row["lease_until"] is None + assert task_row(database, intent)["state"] == "ready" + + +def test_publish_failure_releases_with_backoff_without_secret_log(database, intent, caplog): + transport = Mock() + transport.publish.side_effect = RuntimeError("redis://private-secret") + assert DispatchPublisher(database, transport).publish_batch(1) == 0 + row = rows(database, intent)[0] + assert row["state"] == "pending" and row["attempts"] == 1 + assert row["available_at"] > row["updated_at"] + assert "private-secret" not in caplog.text + + +def test_process_crash_after_send_republishes_same_event(database, intent): + owner = str(uuid4()) + with database.begin() as connection: + row = dispatch.claim(connection, owner) + assert row["tenant_id"] == intent[0] + # Represents broker success followed by publisher kill, before mark-published. + expire(database, intent) + with database.begin() as connection: + assert dispatch.recover_publications(connection) >= 1 + transport = Mock() + assert DispatchPublisher(database, transport).publish_batch(1) == 1 + assert transport.publish.call_args.args[0] == row["id"] + assert rows(database, intent)[0]["attempts"] == 2 + + +@pytest.mark.parametrize("operation", ["renew", "published", "release"]) +@pytest.mark.parametrize("reason", ["expired", "other_owner", "reclaimed"]) +def test_stale_publisher_cannot_mutate_row(database, intent, operation, reason): + owner = str(uuid4()) + with database.begin() as connection: + row = dispatch.claim(connection, owner) + if reason != "other_owner": + expire(database, intent) + if reason == "reclaimed": + with database.begin() as connection: + dispatch.recover_publications(connection) + dispatch.claim(connection, str(uuid4())) + before = rows(database, intent) + with database.begin() as connection: + assert not dispatch.owned_update( + connection, row, str(uuid4()) if reason == "other_owner" else owner, operation + ) + assert rows(database, intent) == before + + +def test_two_publishers_do_not_share_a_live_lease(database, intent): + barrier = Barrier(2) + + def claim(): + with database.begin() as connection: + result = dispatch.claim(connection, str(uuid4())) + barrier.wait(timeout=10) + return result + + with ThreadPoolExecutor(2) as pool: + claims = list(pool.map(lambda _: claim(), range(2))) + assert sum(row is not None and row["tenant_id"] == intent[0] for row in claims) == 1 + assert rows(database, intent)[0]["attempts"] == 1 + + +def test_sweeper_atomic_sequence_and_rate_limit(database, intent): + sweep_dispatch(database) + first = task_row(database, intent) + assert first["dispatch_sequence"] == 2 and first["last_dispatched_at"] is not None + assert len(rows(database, intent)) == 2 + sweep_dispatch(database) + assert task_row(database, intent) == first + assert len(rows(database, intent)) == 2 + last = [ + json.loads(r["payload"]) + for r in rows(database, intent) + if json.loads(r["payload"])["dispatch_sequence"] == 2 + ][0] + assert last["payload"] == json.loads(first["payload"]) + assert last["replay_generation"] == first["replay_generation"] + + +@pytest.mark.parametrize("state", ["leased", "succeeded", "failed", "dead_letter"]) +def test_sweeper_skips_non_dispatchable_tasks(database, intent, state): + with database.begin() as connection: + connection.execute( + text("UPDATE task_runs SET state=:state WHERE id=:id"), + {"state": state, "id": intent[1]}, + ) + sweep_dispatch(database) + assert task_row(database, intent)["dispatch_sequence"] == 1 + assert len(rows(database, intent)) == 1 + + +def test_sweeper_skips_future_retry(database, intent): + with database.begin() as connection: + connection.execute( + text( + "UPDATE task_runs SET state='retry_wait',available_at=" + "UTC_TIMESTAMP(3)+INTERVAL 1 DAY WHERE id=:id" + ), + {"id": intent[1]}, + ) + sweep_dispatch(database) + assert task_row(database, intent)["dispatch_sequence"] == 1 + + +def test_sweeper_concurrency_registers_only_one_new_intent(database, intent): + barrier = Barrier(2) + + def sweep(): + with database.begin() as connection: + dispatch.redispatch(connection) + barrier.wait(timeout=10) + + with ThreadPoolExecutor(2) as pool: + list(pool.map(lambda _: sweep(), range(2))) + assert task_row(database, intent)["dispatch_sequence"] == 2 + assert len(rows(database, intent)) == 2 + + +def test_sweeper_outbox_insert_failure_rolls_back_sequence(database, intent): + before = task_row(database, intent) + + def fail(_conn, _cursor, statement, _parameters, _context, _many): + if statement.startswith("INSERT INTO outbox_events"): + raise RuntimeError("injected") + + event.listen(database, "before_cursor_execute", fail) + try: + with pytest.raises(RuntimeError, match="injected"): + sweep_dispatch(database) + finally: + event.remove(database, "before_cursor_execute", fail) + assert task_row(database, intent) == before + assert len(rows(database, intent)) == 1 + + +@pytest.mark.parametrize("kind", ["version", "tenant", "aggregate", "payload"]) +def test_invalid_outbox_not_sent(database, intent, kind): + row = rows(database, intent)[0] + message = json.loads(row["payload"]) + if kind == "version": + message["schema_version"] = "unknown" + elif kind == "tenant": + message["tenant_id"] = str(uuid4()) + elif kind == "aggregate": + message["task_id"] = str(uuid4()) + else: + message["payload"]["object_key"] = "injected" + with database.begin() as connection: + connection.execute( + text("UPDATE outbox_events SET payload=:payload WHERE id=:id"), + {"id": row["id"], "payload": json.dumps(message)}, + ) + transport = Mock() + assert DispatchPublisher(database, transport).publish_batch(1) == 0 + transport.publish.assert_not_called() + assert rows(database, intent)[0]["state"] == "pending" + + +def test_real_redis_loss_is_repaired_from_published_intent( + database, intent, isolated_redis, monkeypatch +): + port = isolated_redis.connection_pool.connection_kwargs["port"] + monkeypatch.setenv("APP_ENV", "test") + monkeypatch.setenv("REDIS_URL", f"redis://127.0.0.1:{port}/0") + publisher = DispatchPublisher(database, CeleryDispatchTransport()) + app = create_celery_app() + with app.connection_for_read() as connection: + queue = connection.SimpleQueue("q.general") + try: + assert publisher.publish_batch(1) == 1 + notice = queue.get(block=True, timeout=5) + assert notice.payload[0][0]["task_id"] == intent[1] + assert notice.headers["id"] == rows(database, intent)[0]["id"] + notice.ack() # Removes this notice, simulating its loss before execution. + # Redis removes the list key when empty; passive queue_declare then + # returns NOT_FOUND even though deletion/ack succeeded. + assert isolated_redis.llen("q.general") == 0 + assert rows(database, intent)[0]["state"] == "published" + sweep_dispatch(database) + assert publisher.publish_batch(1) == 1 + replacement = queue.get(block=True, timeout=5) + assert replacement.payload[0][0]["task_id"] == intent[1] + assert replacement.payload[0][0]["dispatch_sequence"] == 2 + replacement.ack() + assert task_row(database, intent)["state"] == "ready" + finally: + queue.close() + app.close() diff --git a/tests/persistence/test_image_download_mysql.py b/tests/persistence/test_image_download_mysql.py new file mode 100644 index 0000000..0147fa2 --- /dev/null +++ b/tests/persistence/test_image_download_mysql.py @@ -0,0 +1,332 @@ +"""HTTP/MySQL/Redis grants and local SDK signing; no live object storage or real evidence.""" + +import json +from datetime import timedelta +from unittest.mock import patch +from urllib.parse import parse_qs, urlsplit + +import pytest +from fastapi.testclient import TestClient +from redis.exceptions import ConnectionError +from sqlalchemy import event, text +from sqlalchemy.exc import OperationalError + +from packages.application.image_download import ImageDownloadApplication +from packages.application.job_execution import ImageExecution +from packages.domain.image_validation import ValidatedImage, image_keys +from packages.persistence.image_validation import write_image_result +from packages.persistence.security import revoke_user_sessions, utc_now +from packages.storage.s3 import BUCKET, DownloadGrant +from tests.persistence import test_upload_completion_mysql as completion +from tests.persistence.test_foundations_mysql import count_rows +from tests.persistence.test_identity_mysql import ORIGIN, assert_schema, login +from tests.persistence.test_job_execution_mysql import row, update +from tests.persistence.test_roles_mysql import command, create_user, lab_fixture, role_write +from tests.persistence.test_security_mysql import create_tenant + +upload_api = completion.upload_api +completion_case = completion.completion_case + + +@pytest.fixture +def download_case(completion_case, database): + case = completion_case + with completion.head(case): + case.image = completion.complete(case).json()["data"]["id"] + with database.connect() as connection: + message = json.loads( + connection.scalar( + text( + "SELECT payload FROM outbox_events " + "WHERE tenant_id=:tenant AND event_type='TaskDispatch'" + ), + {"tenant": case.tenant["tenant_id"]}, + ) + ) + execution = ImageExecution(database) + lease = execution.claim(message) + original, analysis = image_keys(lease.tenant_id, lease.input, "b" * 64) + result = ValidatedImage(original, "O/+?=&:v1", analysis, "A/+?=&:v1", "b" * 64, 7, 5) + execution.commit( + lease, + lambda connection, source: write_image_result( + connection, lease.tenant_id, lease.task_id, source, result + ), + ) + case.app.state.image_downloads = ImageDownloadApplication(case.identity, case.storage) + case.result = result + yield case + with database.begin() as connection: + for table in ("task_attempts", "outbox_events", "task_runs"): + connection.execute( + text(f"DELETE FROM {table} WHERE tenant_id=:tenant"), + {"tenant": case.tenant["tenant_id"]}, + ) + + +def download(case, variant=None, http=None): + query = "" if variant is None else f"?variant={variant}" + return (http or case.http).get(f"/api/v1/images/{case.image}/download" + query) + + +def audits(database, case): + with database.connect() as connection: + return [ + dict(value) + for value in connection.execute( + text( + "SELECT * FROM audit_events WHERE tenant_id=:tenant " + "AND action='image.download_original'" + ), + {"tenant": case.tenant["tenant_id"]}, + ).mappings() + ] + + +@pytest.mark.parametrize("variant", [None, "analysis", "original"]) +def test_grant_public_schema_fixed_version_and_original_audit(download_case, database, variant): + case = download_case + old = row(database, "asset_images", case.image) + counts = { + table: count_rows(database, case.tenant, table) + for table in ("uploads", "task_runs", "task_attempts", "outbox_events", "api_idempotency") + } + response = download(case, variant) + assert response.status_code == 200, response.text + assert_schema(response, "DownloadGrant") + assert response.headers["cache-control"] == "no-store" + parts, query = ( + urlsplit(response.json()["data"]["url"]), + parse_qs(urlsplit(response.json()["data"]["url"]).query), + ) + selected = variant or "analysis" + assert parts.netloc == "labsafe.test" and parts.path == f"/{BUCKET}/{old[selected + '_key']}" + assert query["versionId"] == [old[selected + "_object_version"]] + assert query["X-Amz-Expires"] == ["60"] + assert row(database, "asset_images", case.image) == old + assert {table: count_rows(database, case.tenant, table) for table in counts} == counts + recorded = audits(database, case) + assert len(recorded) == int(selected == "original") + if recorded: + audit = recorded[0] + assert audit["actor_id"] == case.tenant["admin_id"] and audit["resource_id"] == case.image + assert audit["request_id"] == response.json()["request_id"] + assert json.loads(audit["changes"]) == {"variant": "original"} + assert "Signature" not in json.dumps(audit, default=str) + + +@pytest.mark.parametrize("role", ["viewer", "inspector", "lab_manager", "remediator"]) +def test_reader_can_get_analysis_but_never_original(download_case, database, role): + case = download_case + user = create_user(case.http, case.session, "download_reader") + assert ( + role_write( + case.http, case.session, user["id"], command(role=role, lab=case.lab["id"]) + ).status_code + == 200 + ) + with TestClient(case.app, base_url=ORIGIN) as http: + login(http, case.tenant, username="download_reader") + assert download(case, http=http).status_code == 200 + with patch.object(case.storage, "presign_image_get") as sign: + assert download(case, "original", http).status_code == 403 + sign.assert_not_called() + assert audits(database, case) == [] + + +def test_foreign_tenant_and_lab_never_sign(download_case, database): + case = download_case + foreign = create_tenant(database, "download_foreign") + lab = lab_fixture(database, case.tenant) + user = create_user(case.http, case.session, "other_lab") + assert ( + role_write(case.http, case.session, user["id"], command(role="viewer", lab=lab)).status_code + == 200 + ) + with patch.object(case.storage, "presign_image_get") as sign: + for tenant, username in ((foreign, "admin"), (case.tenant, "other_lab")): + with TestClient(case.app, base_url=ORIGIN) as http: + login(http, tenant, username=username) + assert download(case, http=http).status_code == 404 + assert download(case, "original", http).status_code == 404 + sign.assert_not_called() + + +@pytest.mark.parametrize("state", ["validating", "rejected", "deleted"]) +def test_non_ready_never_signs_either_variant(download_case, database, state): + case = download_case + update( + database, "UPDATE asset_images SET status=:state WHERE id=:id", state=state, id=case.image + ) + with patch.object(case.storage, "presign_image_get") as sign: + for variant in ("analysis", "original"): + response = download(case, variant) + assert ( + response.status_code == 409 and response.json()["error"]["code"] == "STATE_CONFLICT" + ) + sign.assert_not_called() + assert audits(database, case) == [] + + +@pytest.mark.parametrize( + "field,value,variant", + [ + ("analysis_key", "staging/foreign", "analysis"), + ("analysis_sha256", None, "analysis"), + ("analysis_object_version", "null", "analysis"), + ("original_key", "tenant/foreign/lab/foreign/original/x", "original"), + ("original_object_version", "bad\nversion", "original"), + ], +) +def test_corrupt_ready_reference_never_signs(download_case, database, field, value, variant): + case = download_case + update( + database, f"UPDATE asset_images SET {field}=:value WHERE id=:id", value=value, id=case.image + ) + with patch.object(case.storage, "presign_image_get") as sign: + assert download(case, variant).status_code == 409 + sign.assert_not_called() + + +@pytest.mark.parametrize("kind", ["session", "roles", "image"]) +def test_preflight_is_revalidated_before_signing(download_case, database, kind): + case = download_case + + def limit(actor): + with database.begin() as connection: + if kind == "session": + revoke_user_sessions(connection, actor.tenant_id, actor.user_id) + elif kind == "roles": + connection.execute( + text("DELETE FROM user_roles WHERE tenant_id=:tenant AND user_id=:user"), + {"tenant": actor.tenant_id, "user": actor.user_id}, + ) + else: + connection.execute( + text("UPDATE asset_images SET status='deleted' WHERE id=:id"), + {"id": case.image}, + ) + + with ( + patch.object(case.identity.limits, "download_grant", side_effect=limit), + patch.object(case.storage, "presign_image_get") as sign, + ): + assert download(case, "original").status_code == ( + 401 if kind == "session" else 404 if kind == "roles" else 409 + ) + sign.assert_not_called() + assert audits(database, case) == [] + + +def test_image_and_authorization_locked_during_local_signing(download_case, database): + case = download_case + actual = case.storage.presign_image_get + + def sign(source): + for table, key in (("asset_images", case.image), ("users", case.tenant["admin_id"])): + with database.begin() as other: + with pytest.raises(OperationalError) as caught: + other.execute( + text(f"SELECT id FROM {table} WHERE id=:id FOR UPDATE NOWAIT"), {"id": key} + ) + assert caught.value.orig.args[0] == 3572 + return actual(source) + + with patch.object(case.storage, "presign_image_get", side_effect=sign): + assert download(case, "original").status_code == 200 + with database.begin() as other: + other.execute( + text("SELECT id FROM asset_images WHERE id=:id FOR UPDATE NOWAIT"), {"id": case.image} + ) + assert len(audits(database, case)) == 1 + + +def test_audit_failure_never_returns_original_grant(download_case, database): + case = download_case + before = row(database, "asset_images", case.image) + + def fail(_conn, _cursor, statement, _params, _context, _many): + if statement.startswith("INSERT INTO audit_events"): + raise RuntimeError("private storage credential") + + event.listen(database, "before_cursor_execute", fail) + try: + response = download(case, "original") + finally: + event.remove(database, "before_cursor_execute", fail) + assert response.status_code == 500 and "url" not in response.json() + assert "private storage credential" not in response.text and "X-Amz" not in response.text + assert audits(database, case) == [] and row(database, "asset_images", case.image) == before + + +@pytest.mark.parametrize("seconds", [-1, 61]) +def test_invalid_lifetime_never_returns_grant_or_original_audit(download_case, database, seconds): + case = download_case + grant = DownloadGrant("https://private-url", utc_now() + timedelta(seconds=seconds)) + with patch.object(case.storage, "presign_image_get", return_value=grant): + response = download(case, "original") + assert response.status_code == 503 and "private-url" not in response.text + assert audits(database, case) == [] + + +def test_repeated_original_reads_are_audited_and_later_revocation_blocks_issuance( + download_case, database +): + case = download_case + for _ in range(2): + assert download(case, "original").status_code == 200 + assert len(audits(database, case)) == 2 + update( + database, + "DELETE FROM user_roles WHERE tenant_id=:tenant AND user_id=:user", + tenant=case.tenant["tenant_id"], + user=case.tenant["admin_id"], + ) + with patch.object(case.storage, "presign_image_get") as sign: + assert download(case, "original").status_code == 404 + sign.assert_not_called() + assert len(audits(database, case)) == 2 + + +def test_redis_failure_prevents_download_capability_with_no_fallback(download_case, database): + case = download_case + with ( + patch.object(case.identity.limits.redis, "eval", side_effect=ConnectionError("private")), + patch.object(case.storage, "presign_image_get") as sign, + ): + assert download(case).status_code == 503 + assert download(case, "original").status_code == 503 + sign.assert_not_called() + assert audits(database, case) == [] + + +def test_rate_limit_before_signing_and_outside_transaction(download_case, database): + case = download_case + began = False + + def begin(_connection): + nonlocal began + began = True + + actual = case.identity.limits.download_grant + + def limit(actor): + nonlocal began + # Preflight transaction has closed; begin events during Redis would expose a new DB scope. + began = False + actual(actor) + assert not began + + event.listen(database, "begin", begin) + try: + with patch.object(case.identity.limits, "download_grant", side_effect=limit): + assert download(case).status_code == 200 + finally: + event.remove(database, "begin", begin) + with ( + patch.object(case.identity.limits.redis, "eval", return_value=[121, 10]), + patch.object(case.storage, "presign_image_get") as sign, + ): + response = download(case) + assert response.status_code == 429 and response.headers["retry-after"] == "10" + sign.assert_not_called() diff --git a/tests/persistence/test_image_validation_mysql.py b/tests/persistence/test_image_validation_mysql.py new file mode 100644 index 0000000..d9b4f61 --- /dev/null +++ b/tests/persistence/test_image_validation_mysql.py @@ -0,0 +1,374 @@ +"""Actual image handler + real MySQL; object transport is synthetic unless explicitly stated.""" + +import hashlib +import json +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from pathlib import Path +from threading import Barrier +from unittest.mock import Mock, patch + +import pytest +from jsonschema import Draft202012Validator, FormatChecker +from sqlalchemy import event, text + +from packages.application.image_validation import prepare_image +from packages.application.job_execution import ImageExecution +from packages.domain.image_validation import ( + CONTENT_ERRORS, + RejectedImage, + ValidatedImage, + image_keys, +) +from packages.domain.job_execution import LeaseLost +from packages.domain.security import ServiceError +from packages.persistence.image_validation import write_image_result +from tests.business.test_image_validation import encoded +from tests.persistence.factories import insert +from tests.persistence.test_job_execution_mysql import case, claim, expire, row, update + +# Importing case intentionally shares the same pre-validation graph across F3 tests. +__all__ = ["case"] + + +def outcome(lease): + sha = "b" * 64 + original, analysis = image_keys(lease.tenant_id, lease.input, sha) + return ValidatedImage(original, "original-v1", analysis, "analysis-v1", sha, 7, 5) + + +def writer(lease, result): + return lambda connection, input: write_image_result( + connection, lease.tenant_id, lease.task_id, input, result + ) + + +def events(database, case): + with database.connect() as connection: + return [ + json.loads(v) + for v in connection.scalars( + text( + "SELECT payload FROM outbox_events WHERE tenant_id=:tenant " + "AND event_type='ImageValidated'" + ), + {"tenant": case.tenant}, + ) + ] + + +def audits(database, case): + with database.connect() as connection: + return list( + connection.execute( + text( + "SELECT actor_id,changes,request_id FROM audit_events WHERE tenant_id=:tenant " + "AND action='image.validate'" + ), + {"tenant": case.tenant}, + ).mappings() + ) + + +def test_ready_transaction_registers_evidence_and_valid_event(database, case): + lease = claim(database, case) + result = outcome(lease) + ImageExecution(database).commit(lease, writer(lease, result)) + image = row(database, "asset_images", case.image) + assert image["status"] == row(database, "uploads", case.upload)["status"] == "ready" + assert image["original_key"] == result.original_key + assert image["original_object_version"] == "original-v1" + assert image["analysis_key"] == result.analysis_key + assert image["analysis_object_version"] == "analysis-v1" + assert image["analysis_sha256"] == result.analysis_sha256 + assert (image["width"], image["height"], image["version"]) == ( + 7, + 5, + lease.input.image_version + 1, + ) + assert row(database, "inspection_items", case.item)["status"] == "uploaded" + assert row(database, "inspections", case.inspection)["status"] == "in_progress" + assert row(database, "task_runs", case.job)["state"] == "succeeded" + notices = events(database, case) + assert len(notices) == 1 + schema = json.loads(Path("contracts/events-v1.json").read_text(encoding="utf-8")) + Draft202012Validator(schema, format_checker=FormatChecker()).validate(notices[0]) + assert notices[0]["aggregate_version"] == image["version"] + assert notices[0]["trace_id"] == case.message["trace_id"] + audit = audits(database, case) + assert len(audit) == 1 and audit[0]["actor_id"] is None + assert json.loads(audit[0]["changes"])["status"] == "ready" + with pytest.raises(LeaseLost): + ImageExecution(database).commit(lease, writer(lease, result)) + assert len(events(database, case)) == len(audits(database, case)) == 1 + + +@pytest.mark.parametrize("status", ["uploaded", "needs_retake", "needs_review", "failed"]) +def test_success_preserves_other_capture_states(database, case, status): + update( + database, + "UPDATE inspection_items SET status=:status WHERE id=:id", + id=case.item, + status=status, + ) + lease = claim(database, case) + ImageExecution(database).commit(lease, writer(lease, outcome(lease))) + assert row(database, "inspection_items", case.item)["status"] == status + + +@pytest.mark.parametrize("code", sorted(CONTENT_ERRORS)) +@pytest.mark.parametrize("other_ready", [False, True]) +def test_content_rejection_is_atomic_without_fake_success_event(database, case, code, other_ready): + if other_ready: + with database.begin() as connection: + upload = insert( + connection, + "uploads", + tenant_id=case.tenant, + laboratory_id=case.lab, + inspection_item_id=case.item, + requested_by=case.user, + status="ready", + ) + insert( + connection, + "asset_images", + tenant_id=case.tenant, + laboratory_id=case.lab, + inspection_item_id=case.item, + upload_id=upload, + status="ready", + ) + lease = claim(database, case) + ImageExecution(database).commit(lease, writer(lease, RejectedImage(code))) + image = row(database, "asset_images", case.image) + assert image["status"] == row(database, "uploads", case.upload)["status"] == "rejected" + assert image["analysis_key"] is image["analysis_sha256"] is image["width"] is None + assert row(database, "inspection_items", case.item)["status"] == ( + "uploaded" if other_ready else "draft" + ) + assert row(database, "task_runs", case.job)["state"] == "succeeded" + assert events(database, case) == [] + assert json.loads(audits(database, case)[0]["changes"])["error_code"] == code + + +@pytest.mark.parametrize("status", ["in_progress", "rejected"]) +@pytest.mark.parametrize("ready", [False, True]) +def test_remediation_image_keeps_owner_state(database, case, status, ready): + with database.begin() as connection: + connection.execute( + text("UPDATE remediation_tasks SET status=:status WHERE id=:id"), + {"id": case.task, "status": status}, + ) + for table in ("uploads", "asset_images"): + connection.execute( + text( + f"UPDATE {table} SET inspection_item_id=NULL," + "remediation_task_id=:task WHERE id=:id" + ), + {"task": case.task, "id": case.upload if table == "uploads" else case.image}, + ) + lease = claim(database, case) + assert lease.input.owner_type == "remediation_task" + ImageExecution(database).commit( + lease, writer(lease, outcome(lease) if ready else RejectedImage("IMAGE_INVALID")) + ) + assert row(database, "remediation_tasks", case.task)["status"] == status + assert len(events(database, case)) == int(ready) + + +@pytest.mark.parametrize( + "prefix", + [ + "UPDATE uploads", + "UPDATE inspection_items", + "UPDATE inspections", + "INSERT INTO audit_events", + "INSERT INTO outbox_events", + "UPDATE task_attempts", + ], +) +def test_result_failure_rolls_back_entire_ready_transaction(database, case, prefix): + lease = claim(database, case) + before_image = row(database, "asset_images", case.image) + before_owner = row(database, "inspection_items", case.item) + before_upload = row(database, "uploads", case.upload) + + def fail(_conn, _cursor, statement, _parameters, _context, _many): + if statement.startswith(prefix): + raise RuntimeError("injected") + + event.listen(database, "before_cursor_execute", fail) + try: + with pytest.raises(RuntimeError, match="injected"): + ImageExecution(database).commit(lease, writer(lease, outcome(lease))) + finally: + event.remove(database, "before_cursor_execute", fail) + assert row(database, "asset_images", case.image) == before_image + assert row(database, "uploads", case.upload) == before_upload + assert row(database, "inspection_items", case.item) == before_owner + assert row(database, "inspections", case.inspection)["status"] == "draft" + assert row(database, "task_runs", case.job)["state"] == "leased" + assert events(database, case) == audits(database, case) == [] + + +@pytest.mark.parametrize("kind", ["expired", "cancelled", "superseded_input"]) +def test_prepared_objects_cannot_register_after_fence_invalidated(database, case, kind): + lease = claim(database, case) + result = outcome(lease) + if kind == "expired": + expire(database, case) + elif kind == "cancelled": + update( + database, "UPDATE inspections SET status='cancelled' WHERE id=:id", id=case.inspection + ) + else: + update(database, "UPDATE asset_images SET version=version+1 WHERE id=:id", id=case.image) + with pytest.raises(LeaseLost): + ImageExecution(database).commit(lease, writer(lease, result)) + assert row(database, "asset_images", case.image)["status"] == "validating" + assert events(database, case) == audits(database, case) == [] + + +@pytest.mark.parametrize( + "field,value", + [ + ("analysis_key", "other/key"), + ("original_version", "null"), + ("analysis_sha256", "invalid"), + ("width", 10001), + ("height", True), + ], +) +def test_malformed_server_result_never_commits(database, case, field, value): + lease = claim(database, case) + with pytest.raises((ValueError, ServiceError)): + ImageExecution(database).commit( + lease, writer(lease, replace(outcome(lease), **{field: value})) + ) + assert row(database, "asset_images", case.image)["status"] == "validating" + assert events(database, case) == audits(database, case) == [] + + +def test_two_duplicate_consumers_commit_one_image_result(database, case): + barrier = Barrier(2) + + def run(): + barrier.wait(timeout=5) + return ImageExecution(database).execute( + case.message, + lambda input, cancelled: RejectedImage("IMAGE_INVALID"), + lambda connection, input, result: write_image_result( + connection, case.tenant, case.job, input, result + ), + ) + + with ThreadPoolExecutor(2) as pool: + assert sum(pool.map(lambda _: run(), range(2))) == 1 + assert len(audits(database, case)) == 1 + + +def test_real_decode_to_ready_with_external_storage_outside_transaction(database, case): + payload = encoded() + sha = hashlib.sha256(payload).hexdigest() + update( + database, + "UPDATE uploads SET expected_sha256=:sha,size_bytes=:size WHERE id=:id", + id=case.upload, + sha=sha, + size=len(payload), + ) + update( + database, + "UPDATE asset_images SET original_sha256=:sha WHERE id=:id", + id=case.image, + sha=sha, + ) + storage = Mock() + + def download(*args): + with database.begin() as connection: + connection.execute( + text("SELECT id FROM asset_images WHERE id=:id FOR UPDATE NOWAIT"), + {"id": case.image}, + ) + connection.execute( + text("SELECT id FROM task_runs WHERE id=:id FOR UPDATE NOWAIT"), {"id": case.job} + ) + return payload + + storage.read_staging.side_effect = download + storage.preserve_original.return_value = "original-real-decode-v1" + storage.put_analysis.return_value = "analysis-real-decode-v1" + assert ImageExecution(database).execute( + case.message, + lambda input, cancelled: prepare_image(storage, case.tenant, input), + lambda connection, input, result: write_image_result( + connection, case.tenant, case.job, input, result + ), + ) + assert row(database, "asset_images", case.image)["status"] == "ready" + assert len(events(database, case)) == 1 + + +def test_registered_celery_task_consumes_real_redis_message( + database, case, isolated_redis, monkeypatch +): + from celery.contrib.testing.worker import start_worker + + from apps.worker.app.main import create_celery_app + from apps.worker.dispatch import CeleryDispatchTransport + from packages.application.dispatch import DispatchPublisher + from packages.storage.s3 import S3Settings + from tests.business.test_s3_storage import settings + + port = isolated_redis.connection_pool.connection_kwargs["port"] + monkeypatch.setenv("APP_ENV", "test") + monkeypatch.setenv("REDIS_URL", f"redis://127.0.0.1:{port}/0") + # Publisher child stays dispatch-only; consumer explicitly opts in below. + monkeypatch.setenv("WORKER_IMAGE_VALIDATION_ENABLED", "0") + assert DispatchPublisher(database, CeleryDispatchTransport()).publish_batch(1) == 1 + monkeypatch.setenv("WORKER_IMAGE_VALIDATION_ENABLED", "1") + monkeypatch.setenv("PUBLIC_ORIGIN", "https://labsafe.test") + with patch.object(S3Settings, "from_environment", return_value=settings()): + app = create_celery_app() + try: + with ( + patch("packages.persistence.database.database_engine", return_value=database), + patch( + "apps.worker.image_validation.BoundedImagePrepare", + return_value=lambda input, cancelled: RejectedImage("IMAGE_INVALID"), + ), + start_worker( + app, + pool="solo", + concurrency=1, + queues=["q.general"], + perform_ping_check=False, + loglevel="WARNING", + shutdown_timeout=10, + ), + ): + deadline = time.monotonic() + 15 + while row(database, "task_runs", case.job)["state"] != "succeeded": + assert time.monotonic() < deadline, "Worker failed to consume dispatch" + time.sleep(0.05) + assert row(database, "asset_images", case.image)["status"] == "rejected" + assert len(audits(database, case)) == 1 + finally: + app.close() + + +def test_bare_ready_status_is_not_a_valid_committed_image(database, case): + lease = claim(database, case) + + def incomplete(connection, input): + for table, key in (("asset_images", input.image_id), ("uploads", input.upload_id)): + connection.execute(text(f"UPDATE {table} SET status='ready' WHERE id=:id"), {"id": key}) + + with pytest.raises((ValueError, ServiceError)): + ImageExecution(database).commit(lease, incomplete) + assert row(database, "asset_images", case.image)["status"] == "validating" + assert row(database, "uploads", case.upload)["status"] == "validating" + assert row(database, "task_runs", case.job)["state"] == "leased" diff --git a/tests/persistence/test_inference_execution_mysql.py b/tests/persistence/test_inference_execution_mysql.py new file mode 100644 index 0000000..bb2e5ec --- /dev/null +++ b/tests/persistence/test_inference_execution_mysql.py @@ -0,0 +1,233 @@ +"""Focused MySQL acceptance for the inference execution fence.""" + +import json +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text + +from packages.persistence import inference_execution, rule_execution +from tests.persistence.factories import insert +from tests.persistence.test_identity_mysql import ORIGIN, login +from tests.persistence.test_item_queries_mysql import inspection + + +def post(http, session, path, body): + return http.post( + f"/api/v1{path}", + json=body, + headers={ + "Origin": ORIGIN, + "X-CSRF-Token": session["data"]["csrf_token"], + "Idempotency-Key": str(uuid4()), + }, + ) + + +@pytest.mark.usefixtures("database") +def test_inference_claim_and_facts_commit_are_fenced(identity, database): + tenant, _, app = identity + with TestClient(app, base_url=ORIGIN) as http: + session = login(http, tenant).json() + value, lab, _, _ = inspection(http, session, count=1) + item_id = value["item_ids"][0] + with database.begin() as connection: + upload = insert( + connection, + "uploads", + tenant_id=tenant["tenant_id"], + laboratory_id=lab["id"], + inspection_item_id=item_id, + requested_by=tenant["admin_id"], + object_key="staging/object", + expected_sha256="b" * 64, + mime_type="image/png", + size_bytes=100, + ) + image = insert( + connection, + "asset_images", + tenant_id=tenant["tenant_id"], + laboratory_id=lab["id"], + inspection_item_id=item_id, + upload_id=upload, + status="ready", + original_key="original/object", + original_object_version="original-v1", + original_sha256="b" * 64, + analysis_key="tenant/x/lab/y/analysis/z/" + "a" * 64 + ".png", + analysis_object_version="analysis-v1", + analysis_sha256="a" * 64, + width=100, + height=100, + mime_type="image/png", + ) + connection.execute( + text("UPDATE inspection_items SET status='uploaded' WHERE id=:id"), + {"id": item_id}, + ) + dictionary = insert( + connection, + "dictionary_versions", + tenant_id=tenant["tenant_id"], + status="published", + ) + model = insert( + connection, + "model_versions", + tenant_id=tenant["tenant_id"], + dictionary_version_id=dictionary, + status="published", + ) + rules = insert(connection, "rule_bundles", tenant_id=tenant["tenant_id"]) + rule_set = insert(connection, "rule_sets", tenant_id=tenant["tenant_id"]) + rule_version = insert( + connection, + "rule_versions", + tenant_id=tenant["tenant_id"], + rule_set_id=rule_set, + status="published", + definition=json.dumps( + { + "rule_id": "TEST-DATE", + "scope": "global", + "scope_id": None, + "priority": 1, + "enabled": True, + "effective_from": "2026-01-01T00:00:00Z", + "effective_to": None, + "clauses": [[{"field": "expiry_date", "op": "before_reference_date"}]], + "finding_type": "expired_label", + "severity": "low", + "action_code": "review_date", + "explanation_template": "test", + "source": "test", + "case_ids": ["T-1", "T-2", "T-3"], + } + ), + ) + insert( + connection, + "rule_bundle_members", + tenant_id=tenant["tenant_id"], + bundle_id=rules, + rule_version_id=rule_version, + ) + insert( + connection, + "activations", + tenant_id=tenant["tenant_id"], + laboratory_id=lab["id"], + model_bundle_id=model, + dictionary_version_id=dictionary, + rule_bundle_id=rules, + pipeline_version="vision-v1", + device_profile="cpu", + ) + response = post( + http, + session, + f"/inspection-items/{item_id}/submit", + { + "expected_version": 1, + "images": [{"image_id": image, "role": "overview", "parent_image_id": None}], + }, + ) + assert response.status_code == 202, response.text + run = response.json()["data"] + with database.connect() as connection: + message = json.loads( + connection.scalar( + text("SELECT payload FROM outbox_events WHERE aggregate_id=:task"), + {"task": run["job_id"]}, + ) + ) + with database.begin() as connection: + lease = inference_execution.claim(connection, message, str(uuid4())) + assert lease is not None + result = { + "run_id": lease.input.run_id, + "attempt_id": lease.attempt_id, + "fencing_token": lease.token, + "tenant_id": lease.input.tenant_id, + "request_hash": lease.input.request_hash, + "input_hashes": [ + {"image_id": value.image_id, "sha256": value.sha256} for value in lease.input.images + ], + "model_bundle_id": lease.input.model_bundle_id, + "model_checksum": lease.input.model_checksum, + "dictionary_version_id": lease.input.dictionary_version_id, + "dictionary_sha256": lease.input.dictionary_sha256, + "pipeline_version": "vision-v1", + "outcome": "facts_ready", + "quality": [ + { + "image_id": image, + "status": "pass", + "reasons": [], + "blur_score": 100, + "brightness": 0.5, + "glare_ratio": 0, + } + ], + "detections": [], + "crops": [], + "ocr_fields": [], + "entities": [], + "relations": [], + "timing_ms": { + key: 0 + for key in ( + "download_ms", + "quality_ms", + "detection_ms", + "ocr_ms", + "normalization_ms", + "total_ms", + ) + }, + } + inference_execution.commit_result(connection, lease, result) + state = connection.execute( + text( + "SELECT r.status AS run_status,r.stage,t.state,i.status AS item_status," + "i.current_fact_revision_id " + "FROM inference_runs r JOIN task_runs t ON t.resource_id=r.id " + "AND t.task_type='inference_pipeline' " + "JOIN inspection_items i ON i.id=r.item_id WHERE r.id=:run" + ), + {"run": lease.input.run_id}, + ).one() + assert state.run_status == "processing" + assert state.stage == "rules" + assert state.state == "succeeded" + assert state.item_status == "processing" + assert state.current_fact_revision_id is not None + with database.connect() as connection: + rule_message = json.loads( + connection.scalar( + text( + "SELECT payload FROM outbox_events " + "WHERE JSON_UNQUOTE(JSON_EXTRACT(payload,'$.task_type'))='rule_evaluation' " + "AND tenant_id=:tenant ORDER BY created_at DESC LIMIT 1" + ), + {"tenant": tenant["tenant_id"]}, + ) + ) + with database.begin() as connection: + rule_lease = rule_execution.claim(connection, rule_message, str(uuid4())) + assert rule_lease is not None + assert rule_execution.commit_result(connection, rule_lease) + status = connection.execute( + text( + "SELECT e.status AS evaluation_status,r.status AS run_status," + "i.status AS item_status " + "FROM rule_evaluations e JOIN inference_runs r ON r.id=e.run_id " + "JOIN inspection_items i ON i.id=e.item_id WHERE e.id=:evaluation" + ), + {"evaluation": rule_lease.input.evaluation_id}, + ).one() + assert status.evaluation_status == "completed" + assert status.run_status == "needs_review" + assert status.item_status == "needs_review" diff --git a/tests/persistence/test_inference_submit_mysql.py b/tests/persistence/test_inference_submit_mysql.py new file mode 100644 index 0000000..ee23c38 --- /dev/null +++ b/tests/persistence/test_inference_submit_mysql.py @@ -0,0 +1,131 @@ +"""Real MySQL acceptance for the atomic submitInspectionItem command.""" + +import json +from pathlib import Path +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from jsonschema import Draft202012Validator, FormatChecker +from sqlalchemy import text + +from tests.persistence.factories import insert +from tests.persistence.test_identity_mysql import ORIGIN, login +from tests.persistence.test_item_queries_mysql import inspection + + +def post(http, session, path, body, *, key=None): + headers = { + "Origin": ORIGIN, + "X-CSRF-Token": session["data"]["csrf_token"], + "Idempotency-Key": key or str(uuid4()), + } + return http.post(f"/api/v1{path}", json=body, headers=headers) + + +@pytest.mark.usefixtures("database") +def test_submit_creates_pinned_run_task_and_outbox(identity, database): + tenant, _, app = identity + with TestClient(app, base_url=ORIGIN) as http: + session = login(http, tenant).json() + value, lab, room, _ = inspection(http, session, count=1) + item_id = value["item_ids"][0] + with database.begin() as connection: + upload_id = insert( + connection, + "uploads", + tenant_id=tenant["tenant_id"], + laboratory_id=lab["id"], + inspection_item_id=item_id, + requested_by=tenant["admin_id"], + object_key=f"staging/{tenant['tenant_id']}/{uuid4()}", + expected_sha256="b" * 64, + mime_type="image/png", + size_bytes=100, + ) + image_id = insert( + connection, + "asset_images", + tenant_id=tenant["tenant_id"], + laboratory_id=lab["id"], + inspection_item_id=item_id, + upload_id=upload_id, + status="ready", + original_key=f"original/{tenant['tenant_id']}/{uuid4()}", + original_object_version="original-v1", + original_sha256="b" * 64, + analysis_key=f"analysis/{tenant['tenant_id']}/{uuid4()}", + analysis_object_version="analysis-v1", + analysis_sha256="a" * 64, + width=100, + height=100, + mime_type="image/png", + ) + connection.execute( + text("UPDATE inspection_items SET status='uploaded' WHERE id=:id"), + {"id": item_id}, + ) + dictionary = insert( + connection, "dictionary_versions", tenant_id=tenant["tenant_id"], status="published" + ) + model = insert( + connection, + "model_versions", + tenant_id=tenant["tenant_id"], + dictionary_version_id=dictionary, + status="published", + ) + rules = insert(connection, "rule_bundles", tenant_id=tenant["tenant_id"]) + insert( + connection, + "activations", + tenant_id=tenant["tenant_id"], + laboratory_id=lab["id"], + model_bundle_id=model, + dictionary_version_id=dictionary, + rule_bundle_id=rules, + pipeline_version="vision-v1", + device_profile="cuda", + ) + response = post( + http, + session, + f"/inspection-items/{item_id}/submit", + { + "expected_version": 1, + "images": [{"image_id": image_id, "role": "overview", "parent_image_id": None}], + }, + key="submit-key-1", + ) + assert response.status_code == 202, response.text + run = response.json()["data"] + assert run["status"] == "queued" and run["submission_revision"] == 1 + assert run["input_image_ids"] == [image_id] + assert run["model_bundle_id"] == model and run["dictionary_version_id"] == dictionary + with database.connect() as connection: + item = ( + connection.execute( + text("SELECT status,current_run_id FROM inspection_items WHERE id=:id"), + {"id": item_id}, + ) + .mappings() + .one() + ) + assert item["status"] == "queued" and item["current_run_id"] == run["id"] + task = ( + connection.execute( + text("SELECT task_type,state FROM task_runs WHERE resource_id=:id"), + {"id": run["id"]}, + ) + .mappings() + .one() + ) + assert task == {"task_type": "inference_pipeline", "state": "ready"} + event = connection.scalar( + text("SELECT payload FROM outbox_events WHERE aggregate_id=:id"), + {"id": run["job_id"]}, + ) + spec = json.loads( + (Path(__file__).resolve().parents[2] / "contracts/task-message-v1.json").read_text() + ) + Draft202012Validator(spec, format_checker=FormatChecker()).validate(json.loads(event)) diff --git a/tests/persistence/test_job_api_mysql.py b/tests/persistence/test_job_api_mysql.py new file mode 100644 index 0000000..7959bc7 --- /dev/null +++ b/tests/persistence/test_job_api_mysql.py @@ -0,0 +1,545 @@ +"""Real HTTP/MySQL/Redis replay; precise object HEAD is SDK-stubbed, never live MinIO.""" + +import json +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from pathlib import Path +from threading import Barrier +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from botocore.stub import Stubber +from fastapi.testclient import TestClient +from jsonschema import Draft202012Validator, FormatChecker +from redis.exceptions import ConnectionError +from sqlalchemy import event, text + +from packages.application.job_execution import ImageExecution +from packages.application.jobs import JobApplication +from packages.domain.image_validation import RejectedImage +from packages.domain.job_execution import LeaseLost +from packages.persistence.image_validation import write_image_result +from packages.persistence.security import revoke_user_sessions +from packages.storage.s3 import BUCKET +from tests.persistence import test_upload_completion_mysql as completion +from tests.persistence.test_foundations_mysql import count_rows, post +from tests.persistence.test_identity_mysql import ORIGIN, assert_schema, login +from tests.persistence.test_job_execution_mysql import row, update +from tests.persistence.test_roles_mysql import command, create_user, lab_fixture, role_write +from tests.persistence.test_security_mysql import create_tenant + +upload_api = completion.upload_api +completion_case = completion.completion_case + + +@pytest.fixture +def job_case(completion_case, database): + case = completion_case + with completion.head(case): + image = completion.complete(case).json()["data"] + case.image = image["id"] + with database.connect() as connection: + case.message = json.loads( + connection.scalar( + text( + "SELECT payload FROM outbox_events " + "WHERE tenant_id=:tenant AND event_type='TaskDispatch'" + ), + {"tenant": case.tenant["tenant_id"]}, + ) + ) + case.job = case.message["task_id"] + execution = ImageExecution(database) + case.lease = execution.claim(case.message) + assert execution.fail(case.lease, "INTERNAL_ERROR") + case.version = row(database, "task_runs", case.job)["version"] + case.app.state.jobs = JobApplication(case.identity, case.storage) + yield case + # Exclude only this disposable fixture's notices/tasks from later sweeper tests. + with database.begin() as connection: + for table in ("task_attempts", "outbox_events", "task_runs"): + connection.execute( + text(f"DELETE FROM {table} WHERE tenant_id=:tenant"), + {"tenant": case.tenant["tenant_id"]}, + ) + + +def replay(case, *, key=None, version=None): + return post( + case.http, + case.session, + f"/dead-letters/{case.job}/replay", + {"expected_version": version or case.version, "reason": "Storage recovered"}, + key=key, + ) + + +@contextmanager +def head(case): + with Stubber(case.storage.internal) as stub: + stub.add_response( + "head_object", + {"VersionId": "pinned-v1", "ContentLength": 100, "ContentType": "image/png"}, + {"Bucket": BUCKET, "Key": case.grant["object_key"], "VersionId": "pinned-v1"}, + ) + yield + stub.assert_no_pending_responses() + + +def attempts(database, case): + with database.connect() as connection: + return [ + dict(r) + for r in connection.execute( + text( + "SELECT * FROM task_attempts WHERE tenant_id=:tenant AND task_id=:id " + "ORDER BY replay_generation,attempt" + ), + {"tenant": case.tenant["tenant_id"], "id": case.job}, + ).mappings() + ] + + +def notices(database, case): + with database.connect() as connection: + return [ + json.loads(r) + for r in connection.scalars( + text( + "SELECT payload FROM outbox_events " + "WHERE tenant_id=:tenant AND aggregate_id=:id " + "ORDER BY created_at,id" + ), + {"tenant": case.tenant["tenant_id"], "id": case.job}, + ) + ] + + +def replay_audits(database, case): + with database.connect() as connection: + return list( + connection.execute( + text("SELECT * FROM audit_events WHERE tenant_id=:tenant AND action='job.replay'"), + {"tenant": case.tenant["tenant_id"]}, + ).mappings() + ) + + +def test_job_reads_exact_public_schema_and_no_side_effects(job_case, database): + case = job_case + before = row(database, "task_runs", case.job) + response = case.http.get(f"/api/v1/jobs/{case.job}") + assert response.status_code == 200 + assert_schema(response, "Job") + data = response.json()["data"] + assert data["state"] == "failed" and data["last_error_code"] == "INTERNAL_ERROR" + assert not {"payload", "fencing_token", "lease_owner", "object_key", "tenant_id"} & data.keys() + assert row(database, "task_runs", case.job) == before + assert replay_audits(database, case) == [] + + +@pytest.mark.parametrize( + "state", ["ready", "retry_wait", "leased", "succeeded", "failed", "dead_letter"] +) +def test_job_query_projects_all_persisted_states(job_case, database, state): + case = job_case + update(database, "UPDATE task_runs SET state=:state WHERE id=:id", id=case.job, state=state) + response = case.http.get(f"/api/v1/jobs/{case.job}") + assert response.status_code == 200 and response.json()["data"]["state"] == state + + +def test_terminal_list_filters_lab_and_excludes_other_tasks_and_tenants(job_case, database): + case = job_case + response = case.http.get(f"/api/v1/dead-letters?laboratory_id={case.lab['id']}&page_size=1") + assert response.status_code == 200 + assert_schema(response, "JobPage") + assert response.json()["data"]["total"] == 1 + assert response.json()["data"]["items"][0]["id"] == case.job + assert case.http.get("/api/v1/dead-letters?page=2&page_size=1").json()["data"]["items"] == [] + foreign = create_tenant(database, "foreign_jobs") + with TestClient(case.app, base_url=ORIGIN) as other: + login(other, foreign) + assert other.get("/api/v1/dead-letters").json()["data"]["total"] == 0 + assert other.get(f"/api/v1/jobs/{case.job}").status_code == 404 + assert other.get(f"/api/v1/dead-letters?laboratory_id={case.lab['id']}").status_code == 404 + update( + database, "UPDATE task_runs SET task_type='inference_pipeline' WHERE id=:id", id=case.job + ) + assert case.http.get(f"/api/v1/jobs/{case.job}").status_code == 404 + assert case.http.get("/api/v1/dead-letters").json()["data"]["total"] == 0 + + +@pytest.mark.parametrize("role", ["viewer", "lab_manager", "inspector"]) +def test_read_permission_does_not_grant_admin_replay(job_case, role): + case = job_case + user = create_user(case.http, case.session, "job_reader") + assert ( + role_write( + case.http, case.session, user["id"], command(1, role, case.lab["id"]) + ).status_code + == 200 + ) + with TestClient(case.app, base_url=ORIGIN) as target: + session = login(target, case.tenant, username="job_reader").json() + assert target.get(f"/api/v1/jobs/{case.job}").status_code == 200 + assert target.get("/api/v1/dead-letters").status_code == 403 + with patch.object(case.storage, "inspect_pinned_staging") as inspect: + response = post( + target, + session, + f"/dead-letters/{case.job}/replay", + {"expected_version": case.version, "reason": "not admin"}, + ) + assert response.status_code == 403 + inspect.assert_not_called() + + +@pytest.mark.parametrize("state", ["failed", "dead_letter"]) +def test_replay_preserves_attempts_fences_old_generation_and_enqueues_atomically( + job_case, database, state +): + case = job_case + update(database, "UPDATE task_runs SET state=:state WHERE id=:id", id=case.job, state=state) + old = row(database, "task_runs", case.job) + history = attempts(database, case) + image, upload = ( + row(database, "asset_images", case.image), + row(database, "uploads", case.grant["upload_id"]), + ) + with head(case): + response = replay(case) + assert response.status_code == 202, response.text + assert_schema(response, "Job") + current = row(database, "task_runs", case.job) + assert (current["state"], current["attempt"], current["replay_generation"]) == ("ready", 0, 1) + assert current["fencing_token"] == old["fencing_token"] + 1 + assert current["dispatch_sequence"] == old["dispatch_sequence"] + 1 + assert current["version"] == old["version"] + 1 + assert current["lease_owner"] is current["lease_until"] is current["finished_at"] is None + assert current["payload"] == old["payload"] and current["last_error_code"] is None + assert attempts(database, case) == history + assert row(database, "asset_images", case.image) == image + assert row(database, "uploads", case.grant["upload_id"]) == upload + assert len(replay_audits(database, case)) == 1 + messages = notices(database, case) + assert len(messages) == 2 + latest = next(m for m in messages if m["replay_generation"] == 1) + schema = json.loads(Path("contracts/task-message-v1.json").read_text(encoding="utf-8")) + Draft202012Validator(schema, format_checker=FormatChecker()).validate(latest) + assert ImageExecution(database).claim(case.message) is None + assert not ImageExecution(database).heartbeat(case.lease) + with pytest.raises(LeaseLost): + ImageExecution(database).commit(case.lease, lambda *_: pytest.fail("old lease wrote")) + lease = ImageExecution(database).claim(latest) + assert (lease.generation, lease.attempt, lease.input.object_version) == (1, 1, "pinned-v1") + assert len(attempts(database, case)) == 2 and attempts(database, case)[0] == history[0] + ImageExecution(database).commit( + lease, + lambda connection, source: write_image_result( + connection, lease.tenant_id, lease.task_id, source, RejectedImage("IMAGE_INVALID") + ), + ) + assert row(database, "task_runs", case.job)["state"] == "succeeded" + assert row(database, "asset_images", case.image)["status"] == "rejected" + assert attempts(database, case)[0] == history[0] + assert attempts(database, case)[1]["status"] == "succeeded" + + +def test_same_key_replays_original_response_without_new_head_or_generation(job_case, database): + case, key = job_case, str(uuid4()) + with head(case): + first = replay(case, key=key) + assert first.status_code == 202 + with patch.object( + case.storage, "inspect_pinned_staging", side_effect=AssertionError("no repeated HEAD") + ): + duplicate = replay(case, key=key) + assert duplicate.content == first.content + assert row(database, "task_runs", case.job)["replay_generation"] == 1 + assert len(notices(database, case)) == 2 and len(replay_audits(database, case)) == 1 + assert replay(case, version=case.version).status_code == 409 + + +@pytest.mark.parametrize( + "kind", ["cancelled", "busy", "ready_image", "rejected_image", "wrong_payload", "version"] +) +def test_invalid_replay_source_rejects_before_object_access(job_case, database, kind): + case = job_case + if kind == "cancelled": + update( + database, + "UPDATE inspections SET status='cancelled' WHERE id=:id", + id=case.inspection["id"], + ) + elif kind == "busy": + update( + database, + "UPDATE inspection_items SET status='queued' WHERE id=:id", + id=case.inspection["item_ids"][0], + ) + elif kind.endswith("image"): + update( + database, + "UPDATE asset_images SET status=:state WHERE id=:id", + id=case.image, + state=kind.split("_")[0], + ) + elif kind == "wrong_payload": + update(database, "UPDATE task_runs SET payload='{}' WHERE id=:id", id=case.job) + with patch.object(case.storage, "inspect_pinned_staging") as inspect: + response = replay(case, version=case.version + 1 if kind == "version" else None) + assert response.status_code == 409 + inspect.assert_not_called() + assert len(notices(database, case)) == 1 and replay_audits(database, case) == [] + + +@pytest.mark.parametrize("provider,status", [("NoSuchVersion", 409), ("AccessDenied", 503)]) +def test_missing_pinned_version_or_storage_error_leaves_no_pending_claim( + job_case, database, provider, status +): + case = job_case + before = count_rows(database, case.tenant, "api_idempotency") + old = row(database, "task_runs", case.job) + with Stubber(case.storage.internal) as stub: + stub.add_client_error( + "head_object", + provider, + expected_params={ + "Bucket": BUCKET, + "Key": case.grant["object_key"], + "VersionId": "pinned-v1", + }, + ) + assert replay(case).status_code == status + stub.assert_no_pending_responses() + assert count_rows(database, case.tenant, "api_idempotency") == before + assert row(database, "task_runs", case.job) == old + + +@pytest.mark.parametrize("kind", ["session", "admin", "owner", "input", "upload"]) +def test_head_outside_locks_and_commit_revalidates_mutable_inputs(job_case, database, kind): + case = job_case + + def inspect(tenant, upload_id, pinned): + with database.begin() as connection: + for table, key in (("task_runs", case.job), ("asset_images", case.image)): + connection.execute( + text(f"SELECT id FROM {table} WHERE id=:id FOR UPDATE NOWAIT"), {"id": key} + ) + assert ( + connection.scalar( + text( + "SELECT COUNT(*) FROM api_idempotency " + "WHERE tenant_id=:tenant AND state='pending'" + ), + {"tenant": tenant}, + ) + == 0 + ) + if kind == "session": + revoke_user_sessions(connection, tenant, case.tenant["admin_id"]) + elif kind == "admin": + connection.execute( + text("DELETE FROM user_roles WHERE tenant_id=:tenant AND user_id=:user"), + {"tenant": tenant, "user": case.tenant["admin_id"]}, + ) + elif kind == "owner": + connection.execute( + text("UPDATE inspections SET status='cancelled' WHERE id=:id"), + {"id": case.inspection["id"]}, + ) + else: + connection.execute( + text( + "UPDATE " + + ("uploads" if kind == "upload" else "asset_images") + + " SET version=version+1 WHERE id=:id" + ), + {"id": case.grant["upload_id"] if kind == "upload" else case.image}, + ) + return pinned + + with patch.object(case.storage, "inspect_pinned_staging", side_effect=inspect): + assert replay(case).status_code == ( + 401 if kind == "session" else 403 if kind == "admin" else 409 + ) + assert row(database, "task_runs", case.job)["replay_generation"] == 0 + assert replay_audits(database, case) == [] + + +@pytest.mark.parametrize( + "prefix", + [ + "UPDATE task_runs", + "INSERT INTO outbox_events", + "INSERT INTO audit_events", + "UPDATE api_idempotency", + ], +) +def test_replay_write_failure_rolls_back_every_record(job_case, database, prefix): + case = job_case + old, history = row(database, "task_runs", case.job), attempts(database, case) + count = count_rows(database, case.tenant, "api_idempotency") + + def fail(_conn, _cursor, statement, _parameters, _context, _many): + if statement.startswith(prefix): + raise RuntimeError("injected replay failure") + + event.listen(database, "before_cursor_execute", fail) + try: + with head(case): + response = replay(case) + assert response.status_code == 500 + assert response.json()["error"]["code"] == "INTERNAL_ERROR" + assert "injected" not in response.text + finally: + event.remove(database, "before_cursor_execute", fail) + assert row(database, "task_runs", case.job) == old and attempts(database, case) == history + assert len(notices(database, case)) == 1 and replay_audits(database, case) == [] + assert count_rows(database, case.tenant, "api_idempotency") == count + + +@pytest.mark.parametrize("same_key", [True, False]) +def test_concurrent_replays_create_one_generation(job_case, database, same_key): + case = job_case + barrier = Barrier(2) + key = str(uuid4()) + + def inspect(tenant, upload_id, pinned): + barrier.wait(timeout=10) + return pinned + + def run(index): + with TestClient(case.app, base_url=ORIGIN) as http: + http.cookies.update(case.http.cookies) + return post( + http, + case.session, + f"/dead-letters/{case.job}/replay", + {"expected_version": case.version, "reason": "Storage recovered"}, + key=key if same_key else str(uuid4()), + ) + + with patch.object(case.storage, "inspect_pinned_staging", side_effect=inspect): + with ThreadPoolExecutor(2) as pool: + responses = list(pool.map(run, range(2))) + assert sum(r.status_code == 202 for r in responses) >= 1 + assert all(r.status_code in {202, 409} for r in responses) + assert row(database, "task_runs", case.job)["replay_generation"] == 1 + assert len(notices(database, case)) == 2 and len(replay_audits(database, case)) == 1 + + +def test_redis_write_failure_never_checks_storage_or_replays(job_case, database): + case = job_case + old = row(database, "task_runs", case.job) + with ( + patch.object(case.identity.limits.redis, "eval", side_effect=ConnectionError("secret")), + patch.object(case.storage, "inspect_pinned_staging") as inspect, + ): + assert replay(case).status_code == 503 + inspect.assert_not_called() + assert row(database, "task_runs", case.job) == old + + +def test_same_key_different_body_is_rejected_without_head(job_case, database): + case, key = job_case, str(uuid4()) + with head(case): + assert replay(case, key=key).status_code == 202 + with patch.object(case.storage, "inspect_pinned_staging") as inspect: + response = post( + case.http, + case.session, + f"/dead-letters/{case.job}/replay", + {"expected_version": case.version, "reason": "different request"}, + key=key, + ) + assert response.status_code == 409 + inspect.assert_not_called() + assert row(database, "task_runs", case.job)["replay_generation"] == 1 + + +def test_storage_absence_rolls_back_new_claim_but_allows_completed_response(job_case, database): + case, key = job_case, str(uuid4()) + case.app.state.jobs.storage = None + before = count_rows(database, case.tenant, "api_idempotency") + assert replay(case, key=key).status_code == 503 + assert count_rows(database, case.tenant, "api_idempotency") == before + assert case.http.get(f"/api/v1/jobs/{case.job}").status_code == 200 + case.app.state.jobs.storage = case.storage + with head(case): + first = replay(case, key=key) + assert first.status_code == 202 + case.app.state.jobs.storage = None + assert replay(case, key=key).content == first.content + + +def test_cached_replay_rechecks_admin_after_role_revocation(job_case, database): + case, key = job_case, str(uuid4()) + other = create_user(case.http, case.session, "remaining_admin") + assert ( + role_write(case.http, case.session, other["id"], command(role="safety_admin")).status_code + == 200 + ) + assert ( + role_write( + case.http, + case.session, + case.tenant["admin_id"], + command(role="viewer", lab=case.lab["id"]), + ).status_code + == 200 + ) + case.session = login(case.http, case.tenant).json() + with head(case): + assert replay(case, key=key).status_code == 202 + assert ( + role_write( + case.http, + case.session, + case.tenant["admin_id"], + command(2, "safety_admin"), + revoke=True, + ).status_code + == 200 + ) + assert replay(case, key=key).status_code == 401 + case.session = login(case.http, case.tenant).json() + with patch.object(case.storage, "inspect_pinned_staging") as inspect: + assert replay(case, key=key).status_code == 403 + inspect.assert_not_called() + assert row(database, "task_runs", case.job)["replay_generation"] == 1 + + +def test_job_from_other_laboratory_is_hidden(job_case, database): + case = job_case + lab = lab_fixture(database, case.tenant) + user = create_user(case.http, case.session, "other_lab_reader") + assert ( + role_write(case.http, case.session, user["id"], command(role="viewer", lab=lab)).status_code + == 200 + ) + with TestClient(case.app, base_url=ORIGIN) as http: + login(http, case.tenant, username="other_lab_reader") + assert http.get(f"/api/v1/jobs/{case.job}").status_code == 404 + + +def test_dead_letter_count_and_items_share_snapshot_during_concurrent_replay(job_case, database): + case = job_case + changed = False + + def change_after_count(_conn, _cursor, statement, _params, _context, _many): + nonlocal changed + if not changed and statement.startswith("SELECT COUNT(*) FROM task_runs job"): + changed = True + update(database, "UPDATE task_runs SET state='ready' WHERE id=:id", id=case.job) + + event.listen(database, "after_cursor_execute", change_after_count) + try: + response = case.http.get("/api/v1/dead-letters") + finally: + event.remove(database, "after_cursor_execute", change_after_count) + assert changed and response.status_code == 200 + assert response.json()["data"]["total"] == 1 + assert response.json()["data"]["items"][0]["state"] == "failed" + assert case.http.get("/api/v1/dead-letters").json()["data"]["total"] == 0 diff --git a/tests/persistence/test_job_execution_mysql.py b/tests/persistence/test_job_execution_mysql.py new file mode 100644 index 0000000..381290b --- /dev/null +++ b/tests/persistence/test_job_execution_mysql.py @@ -0,0 +1,493 @@ +"""Real MySQL, synthetic persisted image inputs; no actual image validator.""" + +import json +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from datetime import timedelta +from threading import Barrier +from uuid import uuid4 + +import pytest +from sqlalchemy import event, text + +from packages.application.dispatch import transaction +from packages.application.job_execution import ImageExecution +from packages.domain.job_execution import InvalidDispatch, LeaseLost +from packages.domain.security import ServiceError +from packages.domain.uploads import staging_key +from packages.persistence import job_execution as jobs +from packages.persistence.image_jobs import enqueue_image_validation +from tests.persistence.factories import Graph + + +@pytest.fixture +def case(database): + with database.begin() as connection: + graph = Graph(connection) + # This fixture represents pre-validation capture, before any run selects + # the image. Remove only Graph's synthetic run/image link. + connection.execute( + text("DELETE FROM run_images WHERE tenant_id=:tenant AND image_id=:id"), + {"tenant": graph.tenant, "id": graph.image}, + ) + connection.execute( + text( + "UPDATE uploads SET status='validating',object_key=:key," + "expected_sha256=:sha,size_bytes=10,mime_type='image/png' " + "WHERE id=:id" + ), + {"id": graph.upload, "sha": "a" * 64, "key": staging_key(graph.tenant, graph.upload)}, + ) + connection.execute( + text( + "UPDATE asset_images SET original_key=:key," + "original_object_version='pinned-v1',original_sha256=:sha," + "mime_type='image/png' WHERE id=:id" + ), + {"id": graph.image, "sha": "a" * 64, "key": staging_key(graph.tenant, graph.upload)}, + ) + task = enqueue_image_validation( + connection, + tenant_id=graph.tenant, + upload_id=graph.upload, + image_id=graph.image, + request_id=str(uuid4()), + ) + msg = json.loads( + connection.scalar( + text("SELECT payload FROM outbox_events WHERE aggregate_id=:id"), {"id": task} + ) + ) + graph.job, graph.message = task, msg + yield graph + # Retain the graph in this disposable schema; exclude its task from later + # sweeper tests and never touch unrelated rows from another fixture. + with database.begin() as connection: + connection.execute( + text("DELETE FROM task_attempts WHERE tenant_id=:tenant"), {"tenant": graph.tenant} + ) + connection.execute( + text("DELETE FROM outbox_events WHERE tenant_id=:tenant"), {"tenant": graph.tenant} + ) + connection.execute( + text("DELETE FROM task_runs WHERE tenant_id=:tenant"), {"tenant": graph.tenant} + ) + + +def claim(database, case, message=None): + with transaction(database) as connection: + return jobs.claim(connection, message or case.message, str(uuid4())) + + +def row(database, table, key): + with database.connect() as connection: + return dict( + connection.execute(text(f"SELECT * FROM {table} WHERE id=:id"), {"id": key}) + .mappings() + .one() + ) + + +def update(database, statement, **params): + with database.begin() as connection: + connection.execute(text(statement), params) + + +def expire(database, case): + update( + database, + "UPDATE task_runs SET lease_until=UTC_TIMESTAMP(3)-INTERVAL 1 SECOND WHERE id=:id", + id=case.job, + ) + + +def candidate(database, case): + with transaction(database) as connection: + return next(r for r in jobs.expired_candidates(connection) if r["id"] == case.job) + + +def due(database, case): + update( + database, + "UPDATE task_runs SET available_at=UTC_TIMESTAMP(3)-INTERVAL 1 SECOND WHERE id=:id", + id=case.job, + ) + + +def reject_result(connection, input): + # Synthetic server-owned handler; not the future image validation implementation. + connection.execute( + text("UPDATE asset_images SET status='rejected' WHERE id=:id"), {"id": input.image_id} + ) + connection.execute( + text("UPDATE uploads SET status='rejected' WHERE id=:id"), {"id": input.upload_id} + ) + + +def test_claim_is_atomic_with_attempt_and_database_input(database, case): + lease = claim(database, case) + assert lease.input.object_version == "pinned-v1" and lease.input.expected_sha256 == "a" * 64 + assert lease.input.key == staging_key(case.tenant, case.upload) + task = row(database, "task_runs", case.job) + assert task["attempt"] == task["fencing_token"] == 1 + assert task["lease_until"] - task["heartbeat_at"] == timedelta(seconds=60) + assert task["state"] == "leased" and task["lease_owner"] == lease.owner + with database.connect() as connection: + attempt = ( + connection.execute( + text("SELECT * FROM task_attempts WHERE task_id=:id"), {"id": case.job} + ) + .mappings() + .one() + ) + assert attempt["status"] == "running" and attempt["fencing_token"] == lease.token + assert row(database, "asset_images", case.image)["status"] == "validating" + + +def test_duplicate_consumers_create_one_attempt(database, case): + barrier = Barrier(2) + + def consume(): + barrier.wait(timeout=5) + return claim(database, case) + + with ThreadPoolExecutor(2) as pool: + results = list(pool.map(lambda _: consume(), range(2))) + assert sum(r is not None for r in results) == 1 + assert row(database, "task_runs", case.job)["attempt"] == 1 + + +@pytest.mark.parametrize( + "field,value,invalid", + [ + ("replay_generation", 1, True), + ("dispatch_sequence", 2, True), + ("resource_id", str(uuid4()), True), + ("tenant_id", str(uuid4()), False), + ], +) +def test_message_never_overwrites_persisted_identity(database, case, field, value, invalid): + message = {**case.message, field: value} + if field == "resource_id": + message["payload"] = {**message["payload"], "image_id": value} + if invalid: + with pytest.raises(InvalidDispatch): + claim(database, case, message) + else: + assert claim(database, case, message) is None + assert row(database, "task_runs", case.job)["attempt"] == 0 + + +def test_stale_generation_and_sequence_are_noop(database, case): + update(database, "UPDATE task_runs SET dispatch_sequence=2 WHERE id=:id", id=case.job) + assert claim(database, case) is None + update(database, "UPDATE task_runs SET replay_generation=1 WHERE id=:id", id=case.job) + assert claim(database, case, {**case.message, "dispatch_sequence": 100}) is None + assert row(database, "task_runs", case.job)["attempt"] == 0 + + +@pytest.mark.parametrize("state", ["succeeded", "failed", "dead_letter"]) +def test_terminal_task_ack_has_no_side_effect(database, case, state): + update(database, "UPDATE task_runs SET state=:state WHERE id=:id", id=case.job, state=state) + before = row(database, "task_runs", case.job) + assert claim(database, case) is None + assert row(database, "task_runs", case.job) == before + + +def test_future_retry_not_claimed(database, case): + update( + database, + "UPDATE task_runs SET state='retry_wait',available_at=UTC_TIMESTAMP(3)" + "+INTERVAL 1 DAY WHERE id=:id", + id=case.job, + ) + assert claim(database, case) is None + + +def test_attempt_insert_failure_rolls_back_claim(database, case): + def fail(_conn, _cursor, statement, _parameters, _context, _many): + if statement.startswith("INSERT INTO task_attempts"): + raise RuntimeError("injected attempt failure") + + event.listen(database, "before_cursor_execute", fail) + try: + with pytest.raises(RuntimeError, match="injected"): + claim(database, case) + finally: + event.remove(database, "before_cursor_execute", fail) + assert row(database, "task_runs", case.job)["state"] == "ready" + assert row(database, "task_runs", case.job)["attempt"] == 0 + + +@pytest.mark.parametrize("tamper", ["owner", "token", "generation", "expired"]) +def test_old_lease_cannot_renew_fail_or_commit(database, case, tamper): + lease = claim(database, case) + if tamper == "expired": + expire(database, case) + else: + lease = replace(lease, **{tamper: str(uuid4()) if tamper == "owner" else 99}) + before = row(database, "task_runs", case.job) + with transaction(database) as connection: + assert not jobs.heartbeat(connection, lease) + assert not jobs.fail(connection, lease, "DEPENDENCY_UNAVAILABLE") + with pytest.raises(LeaseLost): + with transaction(database) as connection: + jobs.commit_result(connection, lease, reject_result) + assert row(database, "task_runs", case.job) == before + assert row(database, "asset_images", case.image)["status"] == "validating" + + +def test_renew_checks_domain_and_pinned_version(database, case): + lease = claim(database, case) + with transaction(database) as connection: + assert jobs.heartbeat(connection, lease) + update( + database, + "UPDATE asset_images SET original_object_version='other' WHERE id=:id", + id=case.image, + ) + with transaction(database) as connection: + assert not jobs.heartbeat(connection, lease) + assert row(database, "task_runs", case.job)["last_error_code"] == "LEASE_LOST" + + +@pytest.mark.parametrize("cancelled", [False, True]) +def test_sweeper_abandons_attempt_and_fences_old_worker(database, case, cancelled): + lease = claim(database, case) + if cancelled: + update( + database, "UPDATE inspections SET status='cancelled' WHERE id=:id", id=case.inspection + ) + expire(database, case) + item_before = row(database, "inspection_items", case.item) + discovered = candidate(database, case) + with transaction(database) as connection: + assert jobs.recover(connection, discovered, delay=lambda _: timedelta(seconds=5)) + task = row(database, "task_runs", case.job) + assert task["fencing_token"] == lease.token + 1 + assert task["state"] == ("failed" if cancelled else "retry_wait") + assert row(database, "inspection_items", case.item) == item_before + with database.connect() as connection: + assert ( + connection.scalar( + text("SELECT status FROM task_attempts WHERE task_id=:id"), {"id": case.job} + ) + == "abandoned" + ) + with transaction(database) as connection: + assert not jobs.fail(connection, lease, "INTERNAL_ERROR") + assert not jobs.recover(connection, discovered) + + +def test_sweeper_candidate_stale_after_renewal(database, case): + lease = claim(database, case) + expire(database, case) + discovered = candidate(database, case) + update( + database, + "UPDATE task_runs SET lease_until=UTC_TIMESTAMP(3)+INTERVAL 1 MINUTE WHERE id=:id", + id=case.job, + ) + with transaction(database) as connection: + assert not jobs.recover(connection, discovered) + assert jobs.heartbeat(connection, lease) + + +def test_four_failures_dead_letter_and_no_fifth_attempt(database, case): + for attempt in range(1, 5): + lease = claim(database, case) + assert lease.attempt == attempt + with transaction(database) as connection: + assert jobs.fail( + connection, lease, "DEPENDENCY_UNAVAILABLE", delay=lambda _: timedelta(seconds=5) + ) + task = row(database, "task_runs", case.job) + assert task["state"] == ("retry_wait" if attempt < 4 else "dead_letter") + due(database, case) + assert claim(database, case) is None + with database.connect() as connection: + assert ( + connection.scalar( + text("SELECT COUNT(*) FROM task_attempts WHERE task_id=:id"), {"id": case.job} + ) + == 4 + ) + + +def test_nonretryable_technical_error_stops_immediately(database, case): + lease = claim(database, case) + with transaction(database) as connection: + assert jobs.fail(connection, lease, "INTERNAL_ERROR") + assert row(database, "task_runs", case.job)["state"] == "failed" + + +@pytest.mark.parametrize("code", ["HASH_MISMATCH", "IMAGE_INVALID", "UNKNOWN"]) +def test_content_outcomes_cannot_bypass_domain_handler(database, case, code): + lease = claim(database, case) + with pytest.raises(ValueError): + with transaction(database) as connection: + jobs.fail(connection, lease, code) + assert row(database, "task_runs", case.job)["state"] == "leased" + + +def test_result_commit_is_fenced_atomic_and_duplicate_safe(database, case): + lease = claim(database, case) + service = ImageExecution(database) + service.commit(lease, reject_result) + assert row(database, "task_runs", case.job)["state"] == "succeeded" + assert row(database, "asset_images", case.image)["status"] == "rejected" + assert claim(database, case) is None + with pytest.raises(LeaseLost): + service.commit(lease, reject_result) + + +@pytest.mark.parametrize("failure", ["handler", "empty", "expired"]) +def test_result_failure_rolls_back_all_handler_writes(database, case, failure): + lease = claim(database, case) + + def handler(connection, input): + if failure == "empty": + return + reject_result(connection, input) + if failure == "handler": + raise RuntimeError("handler failed") + connection.execute( + text( + "UPDATE task_runs SET lease_until=UTC_TIMESTAMP(3)-INTERVAL 1 SECOND WHERE id=:id" + ), + {"id": case.job}, + ) + + with pytest.raises((RuntimeError, LeaseLost)): + ImageExecution(database).commit(lease, handler) + assert row(database, "task_runs", case.job)["state"] == "leased" + assert row(database, "asset_images", case.image)["status"] == "validating" + assert row(database, "uploads", case.upload)["status"] == "validating" + + +def test_recovery_service_never_reclaims_live_lease(database, case): + claim(database, case) + ImageExecution(database).recover_expired() + assert row(database, "task_runs", case.job)["state"] == "leased" + expire(database, case) + assert ImageExecution(database).recover_expired() >= 1 + assert row(database, "task_runs", case.job)["state"] == "retry_wait" + + +@pytest.mark.parametrize( + "field,value", + [ + ("status", "queued"), + ("status", "processing"), + ("status", "completed"), + ], +) +def test_busy_owner_prevents_claim_without_business_changes(database, case, field, value): + update( + database, + f"UPDATE inspection_items SET {field}=:value WHERE id=:id", + value=value, + id=case.item, + ) + before = row(database, "inspection_items", case.item) + assert claim(database, case) is None + assert row(database, "task_runs", case.job)["last_error_code"] == "LEASE_LOST" + assert row(database, "inspection_items", case.item) == before + + +@pytest.mark.parametrize("status", ["in_progress", "rejected", "closed", "pending_recheck"]) +def test_remediation_owner_uses_its_own_guard(database, case, status): + update( + database, + "UPDATE remediation_tasks SET status=:status WHERE id=:id", + status=status, + id=case.task, + ) + for table, key in (("uploads", case.upload), ("asset_images", case.image)): + update( + database, + f"UPDATE {table} SET inspection_item_id=NULL,remediation_task_id=:owner WHERE id=:id", + owner=case.task, + id=key, + ) + lease = claim(database, case) + if status in {"in_progress", "rejected"}: + assert lease.input.owner_type == "remediation_task" + else: + assert lease is None + + +def test_concurrent_sweepers_recover_once(database, case): + claim(database, case) + expire(database, case) + discovered = candidate(database, case) + barrier = Barrier(2) + + def recover(): + barrier.wait(timeout=5) + with transaction(database) as connection: + return jobs.recover(connection, discovered) + + with ThreadPoolExecutor(2) as pool: + assert sum(pool.map(lambda _: recover(), range(2))) == 1 + assert row(database, "task_runs", case.job)["fencing_token"] == 2 + + +def test_kill_recovery_exhausts_four_attempts(database, case): + for _ in range(4): + lease = claim(database, case) + expire(database, case) + with transaction(database) as connection: + assert jobs.recover(connection, candidate(database, case)) + due(database, case) + assert row(database, "task_runs", case.job)["state"] == "dead_letter" + with pytest.raises(LeaseLost): + ImageExecution(database).commit(lease, reject_result) + + +def test_attempt_finalization_failure_rolls_back_handler(database, case): + lease = claim(database, case) + + def fail(_conn, _cursor, statement, _parameters, _context, _many): + if statement.startswith("UPDATE task_attempts"): + raise RuntimeError("injected attempt finalize failure") + + event.listen(database, "before_cursor_execute", fail) + try: + with pytest.raises(RuntimeError, match="injected"): + ImageExecution(database).commit(lease, reject_result) + finally: + event.remove(database, "before_cursor_execute", fail) + assert row(database, "task_runs", case.job)["state"] == "leased" + assert row(database, "asset_images", case.image)["status"] == "validating" + + +def test_private_execute_keeps_external_work_outside_locks(database, case): + def prepare(input, cancelled): + assert not cancelled.is_set() + with database.begin() as connection: + connection.execute( + text("SELECT id FROM asset_images WHERE id=:id FOR UPDATE NOWAIT"), + {"id": input.image_id}, + ) + connection.execute( + text("SELECT id FROM task_runs WHERE id=:id FOR UPDATE NOWAIT"), {"id": case.job} + ) + return "synthetic-rejected" + + def write(connection, input, outcome): + assert outcome == "synthetic-rejected" + reject_result(connection, input) + + assert ImageExecution(database).execute(case.message, prepare, write) + assert not ImageExecution(database).execute(case.message, prepare, write) + + +def test_private_execute_technical_failure_preserves_image(database, case): + def unavailable(input, cancelled): + raise ServiceError("DEPENDENCY_UNAVAILABLE", 503, "private URL not logged") + + with pytest.raises(ServiceError): + ImageExecution(database).execute(case.message, unavailable, None) + assert row(database, "task_runs", case.job)["state"] == "retry_wait" + assert row(database, "asset_images", case.image)["status"] == "validating" diff --git a/tests/persistence/test_review_mysql.py b/tests/persistence/test_review_mysql.py new file mode 100644 index 0000000..2e3c0eb --- /dev/null +++ b/tests/persistence/test_review_mysql.py @@ -0,0 +1,183 @@ +"""Focused MySQL acceptance for facts and finding review transactions.""" + +import json +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text + +from tests.persistence.factories import insert +from tests.persistence.test_identity_mysql import ORIGIN, login +from tests.persistence.test_item_queries_mysql import inspection + + +def post(http, session, path, body): + return http.post( + f"/api/v1{path}", + json=body, + headers={ + "Origin": ORIGIN, + "X-CSRF-Token": session["data"]["csrf_token"], + "Idempotency-Key": str(uuid4()), + }, + ) + + +@pytest.mark.usefixtures("database") +def test_finding_decision_and_fact_revision_are_atomic(identity, database): + tenant, _, app = identity + with TestClient(app, base_url=ORIGIN) as http: + session = login(http, tenant).json() + value, lab, _, _ = inspection(http, session, count=1) + item_id = value["item_ids"][0] + with database.begin() as connection: + upload = insert( + connection, + "uploads", + tenant_id=tenant["tenant_id"], + laboratory_id=lab["id"], + inspection_item_id=item_id, + requested_by=tenant["admin_id"], + object_key="staging/review", + expected_sha256="b" * 64, + mime_type="image/png", + size_bytes=100, + ) + image = insert( + connection, + "asset_images", + tenant_id=tenant["tenant_id"], + laboratory_id=lab["id"], + inspection_item_id=item_id, + upload_id=upload, + status="ready", + original_key="original/review", + original_object_version="original-v1", + original_sha256="b" * 64, + analysis_key="analysis/review", + analysis_object_version="analysis-v1", + analysis_sha256="a" * 64, + width=100, + height=100, + mime_type="image/png", + ) + dictionary = insert( + connection, "dictionary_versions", tenant_id=tenant["tenant_id"], status="published" + ) + model = insert( + connection, + "model_versions", + tenant_id=tenant["tenant_id"], + dictionary_version_id=dictionary, + status="published", + ) + bundle = insert(connection, "rule_bundles", tenant_id=tenant["tenant_id"]) + run = insert( + connection, + "inference_runs", + tenant_id=tenant["tenant_id"], + item_id=item_id, + laboratory_id=lab["id"], + model_bundle_id=model, + dictionary_version_id=dictionary, + rule_bundle_id=bundle, + status="needs_review", + stage="done", + result=json.dumps({"detections": []}), + ) + fact = insert( + connection, + "fact_revisions", + tenant_id=tenant["tenant_id"], + item_id=item_id, + run_id=run, + entities="[]", + relations="[]", + dates="[]", + ) + evaluation = insert( + connection, + "rule_evaluations", + tenant_id=tenant["tenant_id"], + item_id=item_id, + run_id=run, + fact_revision_id=fact, + rule_bundle_id=bundle, + status="completed", + ) + finding = insert( + connection, + "findings", + tenant_id=tenant["tenant_id"], + item_id=item_id, + run_id=run, + fact_revision_id=fact, + rule_evaluation_id=evaluation, + evidence=json.dumps({"raw_facts": {"entities": [], "relations": []}}), + ) + insert( + connection, + "run_images", + tenant_id=tenant["tenant_id"], + item_id=item_id, + run_id=run, + image_id=image, + ) + connection.execute( + text( + "UPDATE inspection_items SET status='needs_review',current_run_id=:run," + "current_fact_revision_id=:fact,current_evaluation_id=:evaluation " + "WHERE id=:item" + ), + {"run": run, "fact": fact, "evaluation": evaluation, "item": item_id}, + ) + + response = http.get(f"/api/v1/findings/{finding}") + assert response.status_code == 200, response.text + assert response.json()["data"]["evidence"][0]["image_id"] == image + + response = post( + http, + session, + f"/findings/{finding}/reject", + {"expected_version": 1, "reason": "Reviewed"}, + ) + assert response.status_code == 200, response.text + assert response.json()["data"]["status"] == "rejected" + + with database.connect() as connection: + item_version = connection.scalar( + text("SELECT version FROM inspection_items WHERE id=:id"), {"id": item_id} + ) + response = post( + http, + session, + f"/inspection-items/{item_id}/facts", + { + "expected_version": item_version, + "entities": [], + "relations": [], + "dates": [], + "reason": "No facts require correction", + }, + ) + assert response.status_code == 202, response.text + task_id = response.json()["data"]["id"] + with database.connect() as connection: + state = ( + connection.execute( + text( + "SELECT i.status,i.current_fact_revision_id,i.current_evaluation_id," + "f.superseded_at,t.task_type,t.state " + "FROM inspection_items i JOIN findings f ON f.id=:finding " + "JOIN task_runs t ON t.id=:task WHERE i.id=:item" + ), + {"finding": finding, "task": task_id, "item": item_id}, + ) + .mappings() + .one() + ) + assert state["status"] == "processing" + assert state["superseded_at"] is not None + assert state["task_type"] == "rule_evaluation" and state["state"] == "ready"