From d3f9c3e292ca0b775772d46a3e30880d2c8555e3 Mon Sep 17 00:00:00 2001 From: SayQuan1 <15246906705@163.com> Date: Wed, 7 Oct 2026 18:41:28 +0800 Subject: [PATCH] feat(i-ml01): add COCO80 CPU inference and rebuildable OCR evidence Deliver batches 31-35 with the supplied official ONNX artifacts: preserve all 80 COCO classes, run PP-OCRv6_small on CPU, gate complete image batches with real quality metrics, and rebuild OCR crops from shared recipes. Record artifact/runtime hashes and synthetic replay evidence; keep model weights outside Git. No CUDA or production activation is introduced. Validation: 54 CPU adapter/process/pixel tests passed without skips; real three-image inference reproduced all nine evidence crops including three rotated recognition regions. Contract generation, design validation and Ruff passed; fixture HTTP tests ran in the business test environment. Rollback: revert the model adapter/schema/configuration changes and use the prior development fixture profile. Do not relabel old v4/four-class manifests or evaluation evidence, and do not activate unapproved bundles. --- .github/workflows/ci.yml | 18 +- DEVELOPMENT.md | 89 +- apps/ai_inference/adapters/__init__.py | 37 + apps/ai_inference/adapters/dfine.py | 599 ++++++++ apps/ai_inference/adapters/ocrv6.py | 339 +++++ apps/ai_inference/adapters/quality.py | 104 ++ apps/ai_inference/cpu.py | 243 ++++ apps/ai_inference/detect.py | 57 + apps/ai_inference/ocr.py | 51 + apps/ai_inference/ocr_cpu.py | 142 ++ apps/ai_inference/pipeline.py | 65 + apps/ai_inference/pipeline_cpu.py | 209 +++ contracts/development-acceptance-policy.json | 10 +- contracts/inference-v1.yaml | 93 +- contracts/model-manifest-v1.json | 8 +- contracts/public-api-v1.yaml | 8 +- docs/00-document-control.md | 4 + .../03-ai-inference-process.md | 10 +- docs/02-architecture/04-coding-baseline.md | 2 +- docs/02-architecture/07-python-runtime.md | 2 +- docs/04-ai-rules/01-data-and-rules.md | 10 +- docs/04-ai-rules/02-model-data-plan.md | 97 +- .../03-fact-extraction-contract.md | 6 +- .../01-testing-evaluation.md | 10 +- .../02-deployment-operations.md | 4 +- .../04-acceptance-policy.md | 2 +- docs/08-delivery/01-implementation-plan.md | 20 +- docs/08-delivery/03-design-audit.md | 2 +- docs/08-delivery/04-remediation-plan.md | 2 +- .../31-i-ml01-official-onnx-80class.md | 27 + .../32-cpu-inference-evidence.json | 135 ++ docs/08-delivery/32-i-ml01-cpu-detection.md | 49 + docs/08-delivery/33-i-ml01-ocrv6-cpu.md | 69 + docs/08-delivery/33-ocr-cpu-evidence.json | 187 +++ .../34-i-ml01-quality-cpu-pipeline.md | 60 + .../08-delivery/34-pipeline-cpu-evidence.json | 719 +++++++++ docs/08-delivery/35-crop-cpu-evidence.json | 1281 +++++++++++++++++ .../35-i-ml01-rebuildable-ocr-evidence.md | 46 + docs/08-delivery/README.md | 5 + .../design-validation-results.json | 8 +- packages/image_evidence/__init__.py | 1 + packages/image_evidence/perspective.py | 234 +++ packages/inference_protocol/contract.json | 94 +- pyproject.toml | 12 + requirements/py311-ai-real.txt | 5 + requirements/py311-image-evidence.txt | 4 + tests/ai/test_cpu_pixels.py | 48 + tests/ai/test_cpu_runner.py | 190 +++ tests/ai/test_crop_evidence.py | 230 +++ tests/ai/test_dfine_adapter.py | 187 +++ tests/ai/test_ocrv6_adapter.py | 283 ++++ tests/ai/test_pipeline_cpu.py | 284 ++++ tests/ai/test_quality_pixels.py | 133 ++ tools/design/dfine_reference.py | 274 +++- tools/design/extra_specs.py | 5 +- tools/design/release_reference.py | 6 +- tools/design/release_specs.py | 10 +- tools/design/test_dfine_design.py | 31 +- tools/design/test_release_design.py | 9 +- tools/design/validate_specs.py | 2 +- tools/ml/crop_checks.py | 50 + tools/ml/smoke_cpu.py | 79 + tools/ml/smoke_ocr_cpu.py | 83 ++ tools/ml/smoke_pipeline_cpu.py | 121 ++ 64 files changed, 7040 insertions(+), 164 deletions(-) create mode 100644 apps/ai_inference/adapters/__init__.py create mode 100644 apps/ai_inference/adapters/dfine.py create mode 100644 apps/ai_inference/adapters/ocrv6.py create mode 100644 apps/ai_inference/adapters/quality.py create mode 100644 apps/ai_inference/cpu.py create mode 100644 apps/ai_inference/detect.py create mode 100644 apps/ai_inference/ocr.py create mode 100644 apps/ai_inference/ocr_cpu.py create mode 100644 apps/ai_inference/pipeline.py create mode 100644 apps/ai_inference/pipeline_cpu.py create mode 100644 docs/08-delivery/31-i-ml01-official-onnx-80class.md create mode 100644 docs/08-delivery/32-cpu-inference-evidence.json create mode 100644 docs/08-delivery/32-i-ml01-cpu-detection.md create mode 100644 docs/08-delivery/33-i-ml01-ocrv6-cpu.md create mode 100644 docs/08-delivery/33-ocr-cpu-evidence.json create mode 100644 docs/08-delivery/34-i-ml01-quality-cpu-pipeline.md create mode 100644 docs/08-delivery/34-pipeline-cpu-evidence.json create mode 100644 docs/08-delivery/35-crop-cpu-evidence.json create mode 100644 docs/08-delivery/35-i-ml01-rebuildable-ocr-evidence.md create mode 100644 packages/image_evidence/__init__.py create mode 100644 packages/image_evidence/perspective.py create mode 100644 requirements/py311-ai-real.txt create mode 100644 requirements/py311-image-evidence.txt create mode 100644 tests/ai/test_cpu_pixels.py create mode 100644 tests/ai/test_cpu_runner.py create mode 100644 tests/ai/test_crop_evidence.py create mode 100644 tests/ai/test_dfine_adapter.py create mode 100644 tests/ai/test_ocrv6_adapter.py create mode 100644 tests/ai/test_pipeline_cpu.py create mode 100644 tests/ai/test_quality_pixels.py create mode 100644 tools/ml/crop_checks.py create mode 100644 tools/ml/smoke_cpu.py create mode 100644 tools/ml/smoke_ocr_cpu.py create mode 100644 tools/ml/smoke_pipeline_cpu.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a62e52..a6ded4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,8 +32,8 @@ jobs: - run: python -m unittest discover -s tests/protocol -t . -v - run: python -m pytest tests/business tests/domain tests/security -q - - run: ruff check apps packages tests tools/database - - run: ruff format --check apps packages tests tools/database + - run: ruff check apps packages tests tools/database tools/ml + - run: ruff format --check apps packages tests tools/database tools/ml ai-python311: @@ -55,6 +55,20 @@ jobs: - run: python -m unittest discover -s tests/ai -t . -v - run: python -m unittest discover -s tests/protocol -t . -v + ai-cpu-adapter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: requirements/*.txt + - run: python -m pip install -r requirements/py311-ai-real.txt + - run: python -m pip check + - run: python -m unittest tests.ai.test_dfine_adapter tests.ai.test_cpu_runner tests.ai.test_cpu_pixels tests.ai.test_ocrv6_adapter tests.ai.test_quality_pixels tests.ai.test_pipeline_cpu tests.ai.test_crop_evidence -v + # Official weights remain local; real-model smoke is recorded separately. + design-contracts: runs-on: ubuntu-latest steps: diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5592e74..b4398d6 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -19,7 +19,7 @@ I-02F2 新增 completeUpload/getImage；上传完成受理为 validating，并 I-03A1/A2 接通上传验证任务持久调度、租约、围栏及恢复；I-02F3 接通可选 general Worker、真实 Pillow 图像处理与 ready/rejected 事务。 I-03A3 新增 getJob/listDeadLetters/replayJob 的 validate_image 分支，累计注册 34 个公共业务 API；管理员重放保留固定输入和历史。 I-02F4 新增 downloadImage，累计注册 35 个公共业务 API；ready 分析图按实验室 READ 下载，原图额外 Admin 与审计；签名固定版本、60 秒。 -已实现整改与CSV报告受理/执行/下载；尚不包含 crop 下载、完整巡检业务 API、其他任务类型/事件 inbox/AI 闭环、训练权重或生产部署；真实 MinIO/IAM/HTTPS 验收仍待完成。 +已实现整改与CSV报告受理/执行/下载；尚不包含 crop 下载、完整巡检业务 API、其他任务类型/事件 inbox/AI 闭环或生产部署；真实 MinIO/IAM/HTTPS 验收仍待完成。官方 D-FINE COCO 80 类 ONNX 适配器已加入 `apps/ai_inference/adapters/dfine.py`，独立于当前 fixture HTTP 服务。 未开启身份 API 时，/ready 只检查 I-01A 进程配置；开启后还检查迁移版本和 Redis 连通性，但仍不是完整系统就绪。开启上传也不会使 /ready 检查真实桶、IAM、签名请求或图像验证。 所有后端进程和前端构建均拒绝 production；AI_MODE 只接受 mock。 @@ -37,7 +37,7 @@ py -3.11 -m venv .venv-ai ~~~ Linux 使用 python3.11 创建环境，并将 Scripts/python.exe 替换为 bin/python。 -不需安装模型、CUDA、数据库驱动到 AI 环境；AI 只安装 common.txt 与测试依赖。 +不需安装模型、CUDA、数据库驱动到 fixture AI 环境；真实模型适配器另用 `requirements/py311-ai-real.txt`，避免 fixture CI 安装模型运行时。 requirements 是顶层依赖约束，不是完整传递依赖锁或 GPU 兼容性证明。 ## 3. 配置和启动 @@ -82,7 +82,7 @@ AI 监听 127.0.0.1:8001。五个接口均使用 /internal/inference/v1 前缀 不再提供 /fixture、裸 /version 或未鉴权的 AI /healthz。 请求和响应按生成的契约校验，不给 InferenceResult 私加 is_simulated 字段；模拟身份来自 /version，后续由 Worker 固化到业务 run。 -AI_ALLOWED_TENANTS 必须是显式 UUID 清单；令牌为至少 32 字节的随机 ASCII 字符串。 +AI_ALLOWED_TENANTS 必须是显式 UUID 清单；令牌为至少 32 字节的随机 ASCII 字符串。真实模型适配器的依赖通过 `pip install -e .[ai,ai-real]` 安装，模型、config 和预处理文件须先按 [I-ML-01 验收](docs/08-delivery/31-i-ml01-official-onnx-80class.md) 核验 SHA；当前 fixture 服务仍保持 `AI_MODE=mock`。 请求模型/词典版本、实际 fixture 文件 SHA、设备 cpu、request_hash、图片关联与 analysis key 必须匹配。 fixture 不下载对象或读取图片；质量分数、空事实与阶段耗时均是合成值，不是性能或识别证据。 @@ -371,6 +371,80 @@ Worker 在无数据库连接的子进程内从冻结快照生成 CSV 并上传 I-02I3 已接通 `GET /api/v1/reports/exports/{id}/download`。只有 ready、未过期 CSV 且对象 key/checksum/准确 VersionId/大小证据完整时签发；请求人必须是创建人或完整快照范围的 admin，并在当前事务重新具备每个实验室的 export 权限。签名固定 GET、同源、准确 VersionId 和 60 秒 TTL；不重新生成或读取业务数据，query 参数和跨租户/部分授权请求拒绝。PDF 渲染、完成通知、过期状态转换、精确对象清理、真实存储联调和 UI 仍待后续批次。停用时先停 publisher、排空报告 Worker，再关闭报告开关；保留 schema、对象版本及任务历史，勿在有数据的库执行 downgrade。详见 [报告执行验收](docs/08-delivery/29-i02-report-csv-execution.md) 与 [报告下载验收](docs/08-delivery/30-i02-report-download.md)。 +### 3.16 官方 COCO 80 类 CPU 本地检测（I-ML-01） + +当前使用 CPU，不安装 CUDA。独立 `.venv-ai` 已安装 Python 3.11.4、onnxruntime 1.20.1、numpy 2.2.2 和 Pillow 11.3.0；CPU 依赖为 `requirements/py311-ai-real.txt`。业务环境保持独立。 + +~~~powershell +$env:APP_ENV = 'dev' +.\.venv-ai\Scripts\python.exe -m apps.ai_inference.detect ` + --model C:\Users\12847\Desktop\model.onnx ` + --input C:\path\image1.jpg C:\path\image2.png ` + --output C:\path\new-detections.json +~~~ + +输入换成实际本地路径，支持1–3张PNG/JPEG/WebP；输出父目录需存在、文件需不存在，省略 --output 则写 stdout。默认模型同目录读取 config.json/preprocessor_config.json 并核验 SHA，可用 --config/--preprocessor 指定。默认 --threshold 0.4、--threads 2、--timeout 180，仅 CPUExecutionProvider。阈值是本地开发默认值，尚未现场校准；--run-id 固定 UUID 可复现检测标识。总期限包括子进程启动、模型加载及全部图像，超时回收，任何图失败均不输出部分结果。 + +本地报告为 coco80-local-v1，包含全部80类、输入/模型/runtime摘要、诊断及耗时；明确需要人工复核，OCR/化学实体/容器关系能力不可用。HTTP 服务仍使用 AI_MODE=mock，本命令尚未接 Worker 或事实落库。完整范围、真实合成图证据和后续任务见 [CPU续批验收](docs/08-delivery/32-i-ml01-cpu-detection.md)。 + +### 3.17 PP-OCRv6_small ONNX CPU 本地文字识别 + +用户已选择PP-OCRv6_small。当前det/rec均使用ORT CPU，额外依赖opencv-python-headless4.11.0.86、pyclipper1.3.0.post6已安装到`.venv-ai`，不安装Paddle或CUDA。新环境执行 `python -m pip install -r requirements/py311-ai-real.txt`。 + +~~~powershell +$env:APP_ENV = 'dev' +.\.venv-ai\Scripts\python.exe -m apps.ai_inference.ocr ` + --det-dir D:/LenovoSoftstore/Obsidian-Storage/PP-OCRv6_small_det_onnx ` + --rec-dir D:/LenovoSoftstore/Obsidian-Storage/PP-OCRv6_small_rec_onnx ` + --input C:/path/label.jpg ` + --output C:/path/new-ocr.json +~~~ + +替换图片及输出路径；支持1–3张本地图像，输出父目录需存在、文件需不存在，省略output写UTF-8 stdout。可选text-min=0.6、threads=2、timeout=180、run-id。四个运行制品核验SHA，字符表来自rec YAML。返回原图文字、置信度与归一化TL/TR/BR/BL四边形，低分文字保留并标uncertain；超过100区域整次失败。使用有界spawn并回收超时子进程，无角度分类器。 + +此入口输出ocrv6-local-v2开发报告，需要人工复核；未关联COCO瓶子或生成化学实体/到期日，不作为完整HTTP InferenceResult。原始OCR验收见 [OCR续批](docs/08-delivery/33-i-ml01-ocrv6-cpu.md)，v2裁剪证据见[可重建证据续批](docs/08-delivery/35-i-ml01-rebuildable-ocr-evidence.md)。HTTP仍为AI_MODE=mock。 + +### 3.18 真实质量门禁与联合CPU流水线 + +在已有CPU环境执行质量检查→D-FINE COCO80→PP-OCRv6_small文字识别： + +~~~powershell +$env:APP_ENV = 'dev' +.\.venv-ai\Scripts\python.exe -m apps.ai_inference.pipeline ` + --model C:/Users/12847/Desktop/model.onnx ` + --det-dir D:/LenovoSoftstore/Obsidian-Storage/PP-OCRv6_small_det_onnx ` + --rec-dir D:/LenovoSoftstore/Obsidian-Storage/PP-OCRv6_small_rec_onnx ` + --input C:/path/overview.jpg C:/path/detail.png ` + --output C:/path/new-pipeline.json +.\.venv-ai\Scripts\python.exe -m apps.ai_inference.pipeline ` + --quality-only --input C:/path/overview.jpg --output C:/path/new-quality.json +~~~ + +替换输入/输出路径，支持1–3图、新输出文件，默认blur_min=80/dark_min=0.12/glare_max=0.30（开发阈值）。任一图片质量失败返回整批needs_retake，不加载模型或返回部分检测/OCR；全部通过后各模型加载一次、对原图顺序推理。quality-only无需模型目录，outcome=quality_passed仅表示质量通过；不能据此判断模型ready或业务安全。检查报告outcome，正常补拍评估退出0、技术错误非零退出。默认threads=2/timeout=180，超时回收子进程，仍不安装CUDA。 + +报告cpu-pipeline-local-v2记录实际质量分数、输入/模型/runtime摘要与执行阶段；最多100检测及100文字区域，超限整次失败。文字未关联瓶子、化学实体或日期事实，必须人工复核。HTTP仍为AI_MODE=mock。质量/联合算法见[第34批](docs/08-delivery/34-i-ml01-quality-cpu-pipeline.md)，v2裁剪证据与当前边界见[第35批](docs/08-delivery/35-i-ml01-rebuildable-ocr-evidence.md)。 + +### 3.19 OCR可重建裁剪证据 + +独立OCR和联合入口共用packages/image_evidence/perspective.py。每行crop_evidence记录裁剪配方、独立crop_id、源RGB/PNG/识别像素摘要。PNG保留原图方向；竖长区域的识别90°逆时针旋转单独记录。后续Worker可调用同一函数重建，先核摘要再写PNG；本批未接对象写入或业务落库。以下例子输入必须是与报告一致、已经归一化方向的RGB PNG，路径替换为实际文件： + +~~~python +import json +import numpy as np +from PIL import Image +from packages.image_evidence.perspective import rebuild_ocr_evidence + +with open("pipeline.json", encoding="utf-8") as stream: + report = json.load(stream) +metadata = report["images"][0]["lines"][0]["crop_evidence"] +with Image.open("analysis.png") as image: + pixels = np.asarray(image) +crop = rebuild_ocr_evidence(pixels, metadata) +# crop.png is ready for a future bounded, fenced object-storage write. +~~~ + +当前AI环境已有所需数值依赖；共享依赖独立锁在requirements/py311-image-evidence.txt，未来Worker启用需匹配图像运行环境，且重建SHA必须实际相同。未在业务环境安装NumPy/OpenCV，不改变HTTP fixture依赖隔离。真实三图/9区域跨进程重建和旋转验证见[验收记录](docs/08-delivery/35-i-ml01-rebuildable-ocr-evidence.md)。 + ## 4. 验证 根目录、业务环境执行：领域守卫也使用业务环境，不向 AI 环境引入业务授权或持久化依赖。 @@ -381,8 +455,8 @@ I-02I3 已接通 `GET /api/v1/reports/exports/{id}/download`。只有 ready、 .\.venv-business\Scripts\python.exe -m pytest tests/persistence/test_unit.py --tb=short .\.venv-business\Scripts\python.exe -m pytest tests/business tests/domain tests/security -q -.\.venv-business\Scripts\python.exe -m ruff check apps packages tests tools/database -.\.venv-business\Scripts\python.exe -m ruff format --check apps packages tests tools/database +.\.venv-business\Scripts\python.exe -m ruff check apps packages tests tools/database tools/ml +.\.venv-business\Scripts\python.exe -m ruff format --check apps packages tests tools/database tools/ml ~~~ @@ -391,8 +465,11 @@ AI 环境执行： ~~~powershell .\.venv-ai\Scripts\python.exe -m unittest discover -s tests/ai -t . -v .\.venv-ai\Scripts\python.exe -m unittest discover -s tests/protocol -t . -v +.\.venv-ai\Scripts\python.exe -m unittest tests.ai.test_dfine_adapter tests.ai.test_cpu_runner tests.ai.test_cpu_pixels tests.ai.test_ocrv6_adapter tests.ai.test_quality_pixels tests.ai.test_pipeline_cpu tests.ai.test_crop_evidence -v ~~~ +最后一条须在已安装真实CPU依赖的AI环境执行，覆盖真实NumPy/Pillow及子进程恢复；fixture环境缺这些依赖时的跳过不计为通过。真实权重合成图证据可用 `APP_ENV=test`、`python -m tools.ml.smoke_cpu --model C:\Users\12847\Desktop\model.onnx --output docs/08-delivery/32-cpu-inference-evidence.json` 重建，不需要CUDA，也不证明现场照片准确率。 + I-01A 的业务/AI/协议测试自行在临时目录生成令牌和合成请求，不要求预设 APP_ENV，也不连接 Redis/MySQL/对象存储。 包含实际 API/AI 子进程 HTTP smoke，不只是 import 非空检查。 @@ -417,7 +494,7 @@ CI 必须先 --check，不能先生成来掩盖漂移。设计校验不是应用 I-01B 的实际结果与后续边界见 [验收记录](docs/08-delivery/08-i01b-acceptance.md)。I-01C 会话、RBAC、幂等与租户事务实现及验收范围见 [I-01C 验收记录](docs/08-delivery/09-i01c-acceptance.md)。 -历史 I-03A1/A2、I-02F3/F4、I-03A3 及截至第26批的工作已通过 PR #10 合并，2026-10-07 已 fetch 核验 origin/main=4e4f8bf。第27–30批整改/CSV报告使用 wsq/i-02-remediation-reports；各批验收中的未提交状态是实施当时的历史记录，当前提交/Push/CI状态以本次PR为准。合并前仍需独立审查；数据库/权限/模型变更由两位协作者共同确认，不自动合并。 +历史 I-03A1/A2、I-02F3/F4、I-03A3 及截至第26批的工作已通过 PR #10 合并，2026-10-07 已 fetch 核验 origin/main=4e4f8bf。第27–30批整改/CSV报告使用 wsq/i-02-remediation-reports；第31–35批 CPU 模型与OCR证据使用 wsq/i-ml01-cpu-evidence，后者以报告分支为PR基线，先合并报告PR再调整其基线到main。各批验收中的未提交状态是实施当时的历史记录，当前提交/Push/CI状态以本次PR为准。合并前仍需独立审查；数据库/权限/模型变更由两位协作者共同确认，不自动合并。 I-02A 的领域守卫边界见 [领域命令守卫验收](docs/08-delivery/10-i02a-domain-guards.md)；身份 API、事务和历史故障验证见 [I-02B 验收](docs/08-delivery/11-i02b-identity-api.md)。当前 A1/A2/F3/A3/F4 未推送，不能将旧 PR 的 CI 成绩作为新分支已通过。 I-02C 的角色接口、作用域、版本与会话语义见 [角色 API 验收](docs/08-delivery/12-i02c-role-api.md)。角色请求的 expected_version 来自目标 User.version，laboratory_id 必须显式传 UUID/null；成功改权会撤销目标全部会话，包括操作者修改自身角色时的当前会话。三接口与 I-02B 共用开关和安全请求头，无新增配置。 I-02D 的接口、事务与本轮会话锁修复见 [基础 API 验收](docs/08-delivery/13-i02d-foundation-api.md)。实施计划的 I-0 表格同步记录本地验证状态，不把未提交工作记作远程 CI 或独立批准。 diff --git a/apps/ai_inference/adapters/__init__.py b/apps/ai_inference/adapters/__init__.py new file mode 100644 index 0000000..21b207c --- /dev/null +++ b/apps/ai_inference/adapters/__init__.py @@ -0,0 +1,37 @@ +"""Real-model adapters; they never perform business writes.""" + +from .dfine import ( + ADAPTER_ID, + CLASS_COUNT, + CLASSES, + OFFICIAL_CONFIG_SHA256, + OFFICIAL_MODEL_SHA256, + OFFICIAL_PREPROCESSOR_SHA256, + AdapterError, + ArtifactEvidence, + OnnxDetector, + adapt_outputs, + inspect_artifacts, + postprocess_project, + preprocess_rgb, + require_activation, + validate_session_contract, +) + +__all__ = [ + "ADAPTER_ID", + "AdapterError", + "ArtifactEvidence", + "CLASS_COUNT", + "CLASSES", + "OFFICIAL_CONFIG_SHA256", + "OFFICIAL_MODEL_SHA256", + "OFFICIAL_PREPROCESSOR_SHA256", + "OnnxDetector", + "adapt_outputs", + "inspect_artifacts", + "postprocess_project", + "preprocess_rgb", + "require_activation", + "validate_session_contract", +] diff --git a/apps/ai_inference/adapters/dfine.py b/apps/ai_inference/adapters/dfine.py new file mode 100644 index 0000000..8249815 --- /dev/null +++ b/apps/ai_inference/adapters/dfine.py @@ -0,0 +1,599 @@ +"""Fail-closed adapter for the official D-FINE COCO-80 ONNX export. + +The adapter is deliberately independent from the business application. It only +loads a previously verified artifact, validates the ONNX signature and turns the +300 raw or post-processed queries into bounded detections. COCO labels remain +the model labels; no project-specific four-class remapping is performed. +""" + +import hashlib +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +ADAPTER_ID = "dfine-coco80-rgb-stretch-v1" +MODEL_FAMILY = "dfine_n" +QUERY_COUNT = 300 +CLASS_COUNT = 80 +INPUT_SIZE = (640, 640) +# This ordering is the id2label ordering in the supplied official config.json. +CLASSES = ( + "person", + "bicycle", + "car", + "motorbike", + "aeroplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "sofa", + "pottedplant", + "bed", + "diningtable", + "toilet", + "tvmonitor", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", +) +MAX_DETECTIONS = 100 +OFFICIAL_MODEL_SHA256 = "0f684f409618ee8a822410e754a29caa817d1aa16283ce89cad936d0a48e2f35" +OFFICIAL_CONFIG_SHA256 = "a5c7533f3b72be6bb102b93e1b34ca3643af4e0590408a7881543cbb0aa80c4c" +OFFICIAL_PREPROCESSOR_SHA256 = "cd38cd59999e7a95d68e487fbe5132df3d4e5c32a0836add57e6126ba0c4eaf1" + + +class AdapterError(ValueError): + """Stable adapter validation failure with a public error category.""" + + def __init__(self, code: str, message: str): + self.code = code + super().__init__(message) + + +@dataclass(frozen=True) +class ArtifactEvidence: + model_sha256: str + config_sha256: str + preprocessor_sha256: str + model_bytes: int + config: dict[str, Any] + preprocessor: dict[str, Any] + development_only: bool + activation_allowed: bool + + +def _sha256(path: Path) -> tuple[str, int]: + digest = hashlib.sha256() + size = 0 + try: + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + size += len(chunk) + digest.update(chunk) + except (OSError, TypeError) as error: + raise AdapterError("MODEL_VERSION_UNAVAILABLE", "Model artifact cannot be read") from error + if size < 1: + raise AdapterError("MODEL_VERSION_UNAVAILABLE", "Model artifact is empty") + return digest.hexdigest(), size + + +def _json(path: Path) -> tuple[str, dict[str, Any]]: + try: + raw = path.read_bytes() + value = json.loads(raw.decode("utf-8")) + except (OSError, UnicodeError, ValueError) as error: + raise AdapterError("SCHEMA_MISMATCH", "Model metadata is not valid JSON") from error + if not isinstance(value, dict): + raise AdapterError("SCHEMA_MISMATCH", "Model metadata must be an object") + return hashlib.sha256(raw).hexdigest(), value + + +def inspect_artifacts( + model_path: str | Path, + config_path: str | Path, + preprocessor_path: str | Path, + *, + enforce_known_hash: bool = False, +): + """Hash and inspect supplied official artifacts without loading executable code.""" + model = Path(model_path) + config_digest, config = _json(Path(config_path)) + preprocessor_digest, preprocessor = _json(Path(preprocessor_path)) + model_digest, model_bytes = _sha256(model) + if enforce_known_hash and ( + model_digest != OFFICIAL_MODEL_SHA256 + or config_digest != OFFICIAL_CONFIG_SHA256 + or preprocessor_digest != OFFICIAL_PREPROCESSOR_SHA256 + ): + raise AdapterError( + "MODEL_VERSION_UNAVAILABLE", + "Model artifact hash is not the approved official version", + ) + labels = config.get("id2label") + expected_labels = {str(index): label for index, label in enumerate(CLASSES)} + official_coco = ( + config.get("model_type") == "d_fine" + and config.get("num_queries") == QUERY_COUNT + and labels == expected_labels + and preprocessor.get("size") == {"height": 640, "width": 640} + and preprocessor.get("do_resize") is True + and preprocessor.get("do_rescale") is True + and preprocessor.get("do_pad") is False + and preprocessor.get("do_normalize") is False + and preprocessor.get("rescale_factor") == 1 / 255 + ) + if not official_coco: + raise AdapterError("SCHEMA_MISMATCH", "Unsupported D-FINE artifact metadata") + return ArtifactEvidence( + model_digest, + config_digest, + preprocessor_digest, + model_bytes, + config, + preprocessor, + # The official checkpoint is usable for development/evaluation immediately. + # Production still requires the normal evaluation and release approval gates. + development_only=True, + activation_allowed=True, + ) + + +def require_activation(evidence: ArtifactEvidence) -> None: + if not isinstance(evidence, ArtifactEvidence) or not evidence.activation_allowed: + raise AdapterError( + "MODEL_VERSION_UNAVAILABLE", + "The supplied D-FINE artifact is not approved for activation", + ) + + +def _labels_from_config(evidence: ArtifactEvidence | None) -> tuple[str, ...]: + if evidence is None: + return CLASSES + labels = evidence.config.get("id2label") + if not isinstance(labels, dict): + raise AdapterError("SCHEMA_MISMATCH", "Model config has no id2label map") + try: + resolved = tuple(labels[str(index)] for index in range(CLASS_COUNT)) + except (KeyError, TypeError) as error: + raise AdapterError("SCHEMA_MISMATCH", "Model config label map is incomplete") from error + if resolved != CLASSES: + raise AdapterError("SCHEMA_MISMATCH", "Model config label map does not match COCO-80") + return resolved + + +def preprocess_rgb(image): + """Return direct RGB stretch tensors and ONNX target size ``[H,W]``.""" + try: + import numpy as np + from PIL import Image + except ImportError as error: + raise AdapterError( + "MODEL_VERSION_UNAVAILABLE", "AI runtime dependencies are unavailable" + ) from error + if not isinstance(image, Image.Image) or image.mode != "RGB": + raise AdapterError("SCHEMA_MISMATCH", "Detector input must be an RGB image") + width, height = image.size + if type(width) is not int or type(height) is not int or width < 1 or height < 1: + raise AdapterError("SCHEMA_MISMATCH", "Detector image dimensions are invalid") + resized = image.resize(INPUT_SIZE, Image.Resampling.BILINEAR) + array = np.asarray(resized, dtype=np.uint8).astype(np.float32) / np.float32(255.0) + tensor = np.ascontiguousarray(np.transpose(array, (2, 0, 1))[None, ...], dtype=np.float32) + # RT-DETR/D-FINE post-processing follows the torchvision convention [height,width]. + target_sizes = np.asarray([[height, width]], dtype=np.int64) + return tensor, target_sizes + + +def _finite(value): + return type(value) in (int, float) and math.isfinite(value) + + +def _rows(value, name): + try: + rows = value.tolist() + except AttributeError: + rows = value + if not isinstance(rows, list) or len(rows) != 1 or not isinstance(rows[0], list): + raise AdapterError("SCHEMA_MISMATCH", f"{name} must have batch dimension 1") + return rows[0] + + +def postprocess_project(logits, boxes_cxcywh, original_size, detection_min, *, diagnostics=None): + """Apply qmax/sigmoid/no-NMS semantics to raw COCO-80 query tensors.""" + try: + width, height = original_size + except (TypeError, ValueError): + raise AdapterError("SCHEMA_MISMATCH", "Image size must be [width,height]") from None + if type(width) is not int or type(height) is not int or width < 1 or height < 1: + raise AdapterError("SCHEMA_MISMATCH", "Image size must be positive integers") + if not _finite(detection_min) or not 0 <= detection_min <= 1: + raise AdapterError("VALIDATION_ERROR", "Invalid detection threshold") + logit_rows, box_rows = _rows(logits, "logits"), _rows(boxes_cxcywh, "boxes") + if len(logit_rows) != QUERY_COUNT or len(box_rows) != QUERY_COUNT: + raise AdapterError("SCHEMA_MISMATCH", "Expected exactly 300 detector queries") + result = [] + counts = {"candidates": QUERY_COUNT, "below_threshold": 0, "outside_image": 0} + for query_index, (logit, box) in enumerate(zip(logit_rows, box_rows)): + if not isinstance(logit, list) or not isinstance(box, list): + raise AdapterError("SCHEMA_MISMATCH", "Detector rows must be arrays") + if len(logit) != CLASS_COUNT or len(box) != 4: + raise AdapterError("SCHEMA_MISMATCH", "Expected 80 classes and four box coordinates") + if not all(_finite(v) for v in (*logit, *box)): + raise AdapterError("MODEL_ERROR", "Detector output contains non-finite values") + label = min(range(CLASS_COUNT), key=lambda index: (-logit[index], index)) + score = ( + 1 / (1 + math.exp(-logit[label])) + if logit[label] >= 0 + else math.exp(logit[label]) / (1 + math.exp(logit[label])) + ) + # Official raw exports may emit finite negative widths at very low scores. + # Check finite values for every query, then geometry only for retained queries. + if score < detection_min: + counts["below_threshold"] += 1 + continue + cx, cy, box_width, box_height = box + if box_width <= 0 or box_height <= 0: + raise AdapterError("MODEL_ERROR", "Detector output contains a non-positive box") + xyxy = [ + (cx - box_width / 2) * width, + (cy - box_height / 2) * height, + (cx + box_width / 2) * width, + (cy + box_height / 2) * height, + ] + x1, y1, x2, y2 = xyxy + clipped = [ + max(0, min(width, x1)) / width, + max(0, min(height, y1)) / height, + max(0, min(width, x2)) / width, + max(0, min(height, y2)) / height, + ] + if clipped[2] <= clipped[0] or clipped[3] <= clipped[1]: + counts["outside_image"] += 1 + continue + result.append( + { + "class_id": label, + "type": CLASSES[label], + "bbox": clipped, + "confidence": score, + "query_index": query_index, + } + ) + result.sort( + key=lambda row: ( + row["bbox"][1], + row["bbox"][0], + row["bbox"][3], + row["bbox"][2], + row["class_id"], + row["confidence"], + row["query_index"], + ) + ) + if len(result) > MAX_DETECTIONS: + raise AdapterError("MODEL_ERROR", "Detector result exceeds the 100-detection limit") + if diagnostics is not None: + diagnostics.update(counts, kept=len(result)) + return result + + +def _metadata_shape(value, name): + shape = getattr(value, "shape", None) + dtype = getattr(value, "type", None) + if shape is None or dtype is None: + raise AdapterError("SCHEMA_MISMATCH", f"Missing ONNX metadata for {name}") + return tuple(shape), dtype + + +def validate_session_contract(session) -> str: + """Validate names/shapes and return ``raw`` or ``postprocessed`` mode.""" + try: + inputs = {item.name: item for item in session.get_inputs()} + outputs = {item.name: item for item in session.get_outputs()} + except (AttributeError, TypeError) as error: + raise AdapterError("SCHEMA_MISMATCH", "Invalid ONNX session metadata") from error + if set(inputs) == {"pixel_values"} and set(outputs) == {"logits", "pred_boxes"}: + image_shape, image_type = _metadata_shape(inputs["pixel_values"], "pixel_values") + logits_shape, logits_type = _metadata_shape(outputs["logits"], "logits") + boxes_shape, boxes_type = _metadata_shape(outputs["pred_boxes"], "pred_boxes") + if ( + image_shape not in ((1, 3, 640, 640), ("batch_size", 3, "height", "width")) + or image_type != "tensor(float)" + ): + raise AdapterError("SCHEMA_MISMATCH", "Unexpected pixel_values input signature") + if ( + logits_shape != (image_shape[0], QUERY_COUNT, CLASS_COUNT) + or logits_type != "tensor(float)" + ): + raise AdapterError("SCHEMA_MISMATCH", "Unexpected logits output signature") + if boxes_shape != (image_shape[0], QUERY_COUNT, 4) or boxes_type != "tensor(float)": + raise AdapterError("SCHEMA_MISMATCH", "Unexpected pred_boxes output signature") + return "raw" + if set(inputs) != {"images", "orig_target_sizes"}: + raise AdapterError("SCHEMA_MISMATCH", "Unexpected ONNX input names") + image_shape, image_type = _metadata_shape(inputs["images"], "images") + size_shape, size_type = _metadata_shape(inputs["orig_target_sizes"], "orig_target_sizes") + if image_shape != (1, 3, 640, 640) or image_type != "tensor(float)": + raise AdapterError("SCHEMA_MISMATCH", "Unexpected images input signature") + if size_shape != (1, 2) or size_type != "tensor(int64)": + raise AdapterError("SCHEMA_MISMATCH", "Unexpected orig_target_sizes input signature") + if set(outputs) != {"labels", "boxes", "scores"}: + raise AdapterError("SCHEMA_MISMATCH", "Unexpected ONNX output names") + labels_shape, labels_type = _metadata_shape(outputs["labels"], "labels") + boxes_shape, boxes_type = _metadata_shape(outputs["boxes"], "boxes") + scores_shape, scores_type = _metadata_shape(outputs["scores"], "scores") + if labels_shape != (1, 300) or labels_type != "tensor(int64)": + raise AdapterError("SCHEMA_MISMATCH", "Unexpected labels output signature") + if boxes_shape != (1, 300, 4) or boxes_type != "tensor(float)": + raise AdapterError("SCHEMA_MISMATCH", "Unexpected boxes output signature") + if scores_shape != (1, 300) or scores_type != "tensor(float)": + raise AdapterError("SCHEMA_MISMATCH", "Unexpected scores output signature") + return "postprocessed" + + +def adapt_outputs(labels, boxes_xyxy, scores, original_size, detection_min, *, class_names=None): + """Validate exported labels/boxes/scores before filtering or clipping.""" + try: + width, height = original_size + except (TypeError, ValueError): + raise AdapterError("SCHEMA_MISMATCH", "Image size must be [width,height]") from None + if type(width) is not int or type(height) is not int or width < 1 or height < 1: + raise AdapterError("SCHEMA_MISMATCH", "Image size must be positive integers") + if not _finite(detection_min) or not 0 <= detection_min <= 1: + raise AdapterError("VALIDATION_ERROR", "Invalid detection threshold") + label_rows, box_rows, score_rows = ( + _rows(labels, "labels"), + _rows(boxes_xyxy, "boxes"), + _rows(scores, "scores"), + ) + if not len(label_rows) == len(box_rows) == len(score_rows) == QUERY_COUNT: + raise AdapterError("SCHEMA_MISMATCH", "Expected exactly 300 detector queries") + names = tuple(class_names) if class_names is not None else CLASSES + if len(names) != CLASS_COUNT or any(not isinstance(name, str) or not name for name in names): + raise AdapterError("SCHEMA_MISMATCH", "Detector class map is invalid") + result = [] + for query_index, (label, box, score) in enumerate(zip(label_rows, box_rows, score_rows)): + if type(label) is not int or not 0 <= label < CLASS_COUNT: + raise AdapterError("MODEL_ERROR", "Detector returned an unknown COCO class") + if not isinstance(box, list) or len(box) != 4: + raise AdapterError("SCHEMA_MISMATCH", "Detector box shape is invalid") + if not all(_finite(value) for value in (*box, score)): + raise AdapterError("MODEL_ERROR", "Detector output contains non-finite values") + if not 0 <= score <= 1: + raise AdapterError("MODEL_ERROR", "Detector returned an invalid score") + if score < detection_min: + continue + if box[2] <= box[0] or box[3] <= box[1]: + raise AdapterError("MODEL_ERROR", "Detector returned an invalid box") + x1, y1, x2, y2 = box + clipped = [ + max(0, min(width, x1)) / width, + max(0, min(height, y1)) / height, + max(0, min(width, x2)) / width, + max(0, min(height, y2)) / height, + ] + if clipped[2] <= clipped[0] or clipped[3] <= clipped[1]: + continue + result.append( + { + "class_id": label, + "type": names[label], + "bbox": clipped, + "confidence": score, + "query_index": query_index, + } + ) + result.sort( + key=lambda row: ( + row["bbox"][1], + row["bbox"][0], + row["bbox"][3], + row["bbox"][2], + row["class_id"], + row["confidence"], + row["query_index"], + ) + ) + if len(result) > MAX_DETECTIONS: + raise AdapterError("MODEL_ERROR", "Detector result exceeds the 100-detection limit") + return result + + +class OnnxDetector: + """Small lazy ONNX Runtime wrapper with an explicit provider selection. + + Importing this module does not import onnxruntime. This keeps the fixture + process lightweight while allowing a real worker image to opt into CPU or + CUDA explicitly via ``providers``. + """ + + def __init__(self, session, *, evidence: ArtifactEvidence | None = None): + self._mode = validate_session_contract(session) + self._session = session + self.evidence = evidence + self._class_names = _labels_from_config(evidence) + + @classmethod + def from_path( + cls, + model_path: str | Path, + *, + providers: list[str] | tuple[str, ...] = ("CPUExecutionProvider",), + config_path: str | Path | None = None, + preprocessor_path: str | Path | None = None, + cpu_threads: int = 2, + ) -> "OnnxDetector": + if not providers or any( + not isinstance(provider, str) or not provider for provider in providers + ): + raise AdapterError( + "VALIDATION_ERROR", "At least one explicit ONNX provider is required" + ) + if type(cpu_threads) is not int or not 1 <= cpu_threads <= 16: + raise AdapterError("VALIDATION_ERROR", "CPU threads must be 1..16") + model_path = Path(model_path) + evidence = inspect_artifacts( + model_path, + config_path or model_path.with_name("config.json"), + preprocessor_path or model_path.with_name("preprocessor_config.json"), + enforce_known_hash=True, + ) + require_activation(evidence) + try: + import onnxruntime as ort + except ImportError as error: + raise AdapterError( + "MODEL_VERSION_UNAVAILABLE", "ONNX Runtime is unavailable" + ) from error + try: + available = set(ort.get_available_providers()) + if any(provider not in available for provider in providers): + raise AdapterError( + "MODEL_VERSION_UNAVAILABLE", "Requested ONNX provider is unavailable" + ) + # Load the exact bytes verified here, avoiding a path replacement between + # hashing and ONNX Runtime loading the model. + with model_path.open("rb") as stream: + model_bytes = stream.read(evidence.model_bytes + 1) + if hashlib.sha256(model_bytes).hexdigest() != evidence.model_sha256: + raise AdapterError("HASH_MISMATCH", "Model artifact changed during loading") + options = ort.SessionOptions() + options.intra_op_num_threads = cpu_threads + options.inter_op_num_threads = 1 + options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + session = ort.InferenceSession( + model_bytes, sess_options=options, providers=list(providers) + ) + disable_fallback = getattr(session, "disable_fallback", None) + if not callable(disable_fallback): + raise AdapterError( + "MODEL_VERSION_UNAVAILABLE", "ONNX fallback control is unavailable" + ) + disable_fallback() + if tuple(session.get_providers())[: len(providers)] != tuple(providers): + raise AdapterError( + "MODEL_VERSION_UNAVAILABLE", "ONNX provider order was not honored" + ) + except AdapterError: + raise + except Exception as error: # runtime errors must not leak filesystem details + raise AdapterError( + "MODEL_VERSION_UNAVAILABLE", "ONNX model could not be loaded" + ) from error + return cls(session, evidence=evidence) + + @property + def providers(self) -> tuple[str, ...]: + try: + return tuple(self._session.get_providers()) + except AttributeError as error: + raise AdapterError("SCHEMA_MISMATCH", "Invalid ONNX session") from error + + def detect( + self, image, detection_min: float = 0.4, *, diagnostics=None + ) -> list[dict[str, Any]]: + if not _finite(detection_min) or not 0 <= detection_min <= 1: + raise AdapterError("VALIDATION_ERROR", "Invalid detection threshold") + tensor, target_sizes = preprocess_rgb(image) + width, height = image.size + try: + if self._mode == "raw": + outputs = self._session.run(["logits", "pred_boxes"], {"pixel_values": tensor}) + else: + outputs = self._session.run( + ["labels", "boxes", "scores"], + {"images": tensor, "orig_target_sizes": target_sizes}, + ) + except Exception as error: + raise AdapterError("MODEL_ERROR", "ONNX inference failed") from error + expected_count = 2 if self._mode == "raw" else 3 + if not isinstance(outputs, (list, tuple)) or len(outputs) != expected_count: + raise AdapterError("SCHEMA_MISMATCH", "ONNX output count is invalid") + shapes = ( + ((1, QUERY_COUNT, CLASS_COUNT), (1, QUERY_COUNT, 4)) + if self._mode == "raw" + else ((1, QUERY_COUNT), (1, QUERY_COUNT, 4), (1, QUERY_COUNT)) + ) + types = ("float32", "float32") if self._mode == "raw" else ("int64", "float32", "float32") + for output, shape, dtype in zip(outputs, shapes, types): + if ( + tuple(getattr(output, "shape", ())) != shape + or str(getattr(output, "dtype", None)) != dtype + ): + raise AdapterError("SCHEMA_MISMATCH", "ONNX output tensor signature is invalid") + if self._mode == "raw": + return postprocess_project( + outputs[0], outputs[1], [width, height], detection_min, diagnostics=diagnostics + ) + return adapt_outputs( + outputs[0], + outputs[1], + outputs[2], + [width, height], + detection_min, + class_names=self._class_names, + ) diff --git a/apps/ai_inference/adapters/ocrv6.py b/apps/ai_inference/adapters/ocrv6.py new file mode 100644 index 0000000..647bb3a --- /dev/null +++ b/apps/ai_inference/adapters/ocrv6.py @@ -0,0 +1,339 @@ +"""Local PP-OCRv6_small ONNX adapter; CPU only, no Paddle or downloads.""" + +import hashlib +import math +from pathlib import Path + +from apps.ai_inference.adapters.dfine import AdapterError +from packages.image_evidence.perspective import ( + CropError, + CropTransform, + rebuild_crop, + recognition_rgb, + rgb_digest, +) + +ADAPTER_ID = "ppocrv6-small-db-ctc-cpu-v2" +ARTIFACTS = { + "det": { + "inference.onnx": "d73e0058b7a8086bbd57f3d10b8bcd4ff95363f67e06e2762b5e814fe9c9410e", + "inference.yml": "193f435274bf9f0b5f71a929bbfbcf148282df7e633b34e7c373e8f44741b516", + }, + "rec": { + "inference.onnx": "5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634", + "inference.yml": "ab078671bb49f06228eadccd34f1bb501e157f7a047095ffb943ba81512c77d1", + }, +} +MAX_REGIONS = 100 +MAX_TEXT = 2000 +DET_MAX_SIDE = 960 +REC_MAX_WIDTH = 3200 + + +def load_artifacts(directory, role): + import yaml + + blobs = {} + for name, expected in ARTIFACTS[role].items(): + try: + with (Path(directory) / name).open("rb") as stream: + raw = stream.read(32 * 1024 * 1024 + 1) + except OSError as error: + raise AdapterError("MODEL_VERSION_UNAVAILABLE", "OCR artifact unavailable") from error + if hashlib.sha256(raw).hexdigest() != expected: + raise AdapterError("HASH_MISMATCH", "OCR artifact hash mismatch") + blobs[name] = raw + config = yaml.safe_load(blobs["inference.yml"]) + if config["Global"]["model_name"] != f"PP-OCRv6_small_{role}": + raise AdapterError("SCHEMA_MISMATCH", "OCR model identity mismatch") + return blobs["inference.onnx"], config + + +def validate_session(session, role): + inputs, outputs = session.get_inputs(), session.get_outputs() + if len(inputs) != 1 or len(outputs) != 1: + raise AdapterError("SCHEMA_MISMATCH", "OCR must have one input and one output") + source, result = inputs[0], outputs[0] + if ( + source.name != "x" + or result.name != "fetch_name_0" + or source.type != "tensor(float)" + or result.type != "tensor(float)" + or len(source.shape) != 4 + or source.shape[1] != 3 + ): + raise AdapterError("SCHEMA_MISMATCH", "OCR tensor signature mismatch") + if role == "det": + valid = len(result.shape) == 4 and result.shape[1] == 1 + else: + valid = source.shape[2] == 48 and len(result.shape) == 3 and result.shape[2] == 18710 + if not valid: + raise AdapterError("SCHEMA_MISMATCH", "OCR output signature mismatch") + if session.get_providers() != ["CPUExecutionProvider"]: + raise AdapterError("MODEL_VERSION_UNAVAILABLE", "OCR requires a CPU-only session") + + +def _probabilities(array): + import numpy as np + + if not isinstance(array, np.ndarray) or array.dtype != np.float32: + raise AdapterError("SCHEMA_MISMATCH", "OCR output must be float32") + if not np.isfinite(array).all() or np.any(array < 0) or np.any(array > 1): + raise AdapterError("MODEL_ERROR", "OCR returned invalid probabilities") + + +def decode_ctc(probs, characters): + import numpy as np + + _probabilities(probs) + if probs.ndim != 3 or probs.shape[0] != 1 or probs.shape[1] < 1: + raise AdapterError("SCHEMA_MISMATCH", "Invalid CTC batch or sequence") + if probs.shape[2] != len(characters): + raise AdapterError("SCHEMA_MISMATCH", "CTC vocabulary mismatch") + if not np.allclose(probs.sum(axis=2), 1, atol=1e-4, rtol=1e-4): + raise AdapterError("MODEL_ERROR", "CTC probabilities are not normalized") + indices = probs[0].argmax(axis=1) + text, scores, previous = [], [], -1 + for step, index in enumerate(indices): + index = int(index) + if index != 0 and index != previous: + text.append(characters[index]) + scores.append(float(probs[0, step, index])) + previous = index + decoded = "".join(text) + if len(decoded) > MAX_TEXT: + raise AdapterError("MODEL_ERROR", "OCR text exceeds the capacity limit") + return decoded, sum(scores) / len(scores) if scores else 0.0 + + +def det_tensor(rgb): + import cv2 + import numpy as np + + height, width = rgb.shape[:2] + ratio = min(1.0, DET_MAX_SIDE / max(height, width)) + out_h = max(32, int(round(height * ratio / 32)) * 32) + out_w = max(32, int(round(width * ratio / 32)) * 32) + bgr = cv2.resize(rgb[:, :, ::-1], (out_w, out_h), interpolation=cv2.INTER_LINEAR) + pixels = bgr.astype(np.float32) / np.float32(255) + pixels = (pixels - np.array([0.485, 0.456, 0.406], dtype=np.float32)) / np.array( + [0.229, 0.224, 0.225], dtype=np.float32 + ) + return np.ascontiguousarray(pixels.transpose(2, 0, 1)[None]) + + +def rec_tensor(rgb): + import cv2 + import numpy as np + + height, width = rgb.shape[:2] + target = max(320, math.ceil(48 * width / height / 8) * 8) + if target > REC_MAX_WIDTH: + raise AdapterError("MODEL_ERROR", "OCR line exceeds the recognition width limit") + resized_w = min(target, math.ceil(48 * width / height)) + bgr = cv2.resize(rgb[:, :, ::-1], (resized_w, 48), interpolation=cv2.INTER_LINEAR) + pixels = bgr.astype(np.float32).transpose(2, 0, 1) / np.float32(127.5) - np.float32(1) + padded = np.zeros((1, 3, 48, target), dtype=np.float32) + padded[0, :, :, :resized_w] = pixels + return padded + + +def _quad(points): + import numpy as np + + points = np.asarray(points, dtype=np.float32).reshape(4, 2) + left_right = points[np.argsort(points[:, 0], kind="stable")] + left = left_right[:2][np.argsort(left_right[:2, 1], kind="stable")] + right = left_right[2:][np.argsort(left_right[2:, 1], kind="stable")] + return np.array([left[0], right[0], right[1], left[1]], dtype=np.float32) + + +def db_regions(probs, original_size): + import cv2 + import numpy as np + import pyclipper + + _probabilities(probs) + if probs.ndim != 4 or probs.shape[:2] != (1, 1) or min(probs.shape[2:]) < 1: + raise AdapterError("SCHEMA_MISMATCH", "Invalid DB probability map") + width, height = original_size + bitmap = probs[0, 0] + contours, _ = cv2.findContours( + (bitmap > 0.2).astype(np.uint8), cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE + ) + if len(contours) > 3000: + raise AdapterError("MODEL_ERROR", "OCR contour capacity exceeded") + result = [] + for contour in contours: + rect = cv2.minAreaRect(contour) + if min(rect[1]) < 3: + continue + points = _quad(cv2.boxPoints(rect)) + x0, y0 = np.floor(points.min(axis=0)).astype(int) + x1, y1 = np.ceil(points.max(axis=0)).astype(int) + x0, x1 = max(0, x0), min(bitmap.shape[1] - 1, x1) + y0, y1 = max(0, y0), min(bitmap.shape[0] - 1, y1) + mask = np.zeros((y1 - y0 + 1, x1 - x0 + 1), dtype=np.uint8) + cv2.fillPoly(mask, [np.round(points - [x0, y0]).astype(np.int32)], 1) + score = float(cv2.mean(bitmap[y0 : y1 + 1, x0 : x1 + 1], mask)[0]) + if score < 0.45: + continue + perimeter = cv2.arcLength(points, True) + distance = cv2.contourArea(points) * 1.4 / perimeter + offset = pyclipper.PyclipperOffset() + offset.AddPath( + np.round(points * 1024).astype(np.int64).tolist(), + pyclipper.JT_ROUND, + pyclipper.ET_CLOSEDPOLYGON, + ) + expanded = offset.Execute(distance * 1024) + if len(expanded) != 1: + continue + rect = cv2.minAreaRect(np.asarray(expanded[0], dtype=np.float32) / 1024) + if min(rect[1]) < 5: + continue + points = _quad(cv2.boxPoints(rect)) + points[:, 0] = np.clip(np.round(points[:, 0] / bitmap.shape[1] * width), 0, width - 1) + points[:, 1] = np.clip(np.round(points[:, 1] / bitmap.shape[0] * height), 0, height - 1) + if cv2.contourArea(points) < 1: + continue + result.append({"quad_pixels": points, "detection_confidence": score}) + if len(result) > MAX_REGIONS: + raise AdapterError("MODEL_ERROR", "OCR region capacity exceeded") + result.sort( + key=lambda r: tuple(r["quad_pixels"].min(axis=0)[::-1]) + tuple(r["quad_pixels"].flatten()) + ) + return result + + +def crop_with_evidence(rgb, points): + import cv2 + import numpy as np + + points = np.asarray(points, dtype=np.float32) + if ( + points.shape != (4, 2) + or not np.isfinite(points).all() + or not cv2.isContourConvex(points) + or cv2.contourArea(points) < 1 + ): + raise AdapterError("MODEL_ERROR", "OCR crop quadrilateral is invalid") + tl, tr, br, bl = points + width = int(max(np.linalg.norm(tl - tr), np.linalg.norm(bl - br))) + height = int(max(np.linalg.norm(tl - bl), np.linalg.norm(tr - br))) + if min(width, height) < 2 or max(width, height) > 2048: + raise AdapterError("MODEL_ERROR", "OCR crop dimensions are invalid") + source_h, source_w = rgb.shape[:2] + normalized = points.astype(np.float64) / [max(1, source_w - 1), max(1, source_h - 1)] + recipe = CropTransform(tuple(map(tuple, normalized.tolist())), width, height) + rotation = 90 if height / width >= 1.5 else 0 + try: + # Recognition is derived from the serialized recipe, never a separate warp. + evidence = rebuild_crop(rgb, recipe) + crop = recognition_rgb(evidence.pixels, rotation) + except CropError as error: + raise AdapterError(error.code, "OCR evidence crop failed") from error + return crop, { + "recipe": recipe.as_dict(), + "png_sha256": evidence.sha256, + "png_size_bytes": len(evidence.png), + "recognition_rotation_ccw": rotation, + "recognition_width": crop.shape[1], + "recognition_height": crop.shape[0], + "recognition_rgb_sha256": rgb_digest(crop), + } + + +def crop_region(rgb, points): + return crop_with_evidence(rgb, points)[0] + + +class OnnxOCR: + def __init__(self, det, rec, characters): + validate_session(det, "det") + validate_session(rec, "rec") + if len(characters) != 18710 or characters[0] != "" or characters[-1] != " ": + raise AdapterError("SCHEMA_MISMATCH", "Unexpected CTC dictionary") + self.det, self.rec, self.characters = det, rec, characters + + @classmethod + def from_directories(cls, det_dir, rec_dir, threads=2): + import onnxruntime as ort + + if type(threads) is not int or not 1 <= threads <= 16: + raise AdapterError("VALIDATION_ERROR", "Invalid OCR CPU threads") + det_bytes, det_config = load_artifacts(det_dir, "det") + rec_bytes, rec_config = load_artifacts(rec_dir, "rec") + if det_config["PostProcess"] != { + "name": "DBPostProcess", + "thresh": 0.2, + "box_thresh": 0.45, + "max_candidates": 3000, + "unclip_ratio": 1.4, + }: + raise AdapterError("SCHEMA_MISMATCH", "Unsupported DB parameters") + chars = ("", *rec_config["PostProcess"]["character_dict"], " ") + if len(set(chars)) != len(chars): + raise AdapterError("SCHEMA_MISMATCH", "Duplicate CTC dictionary entries") + options = ort.SessionOptions() + options.intra_op_num_threads = threads + options.inter_op_num_threads = 1 + options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + try: + det = ort.InferenceSession( + det_bytes, sess_options=options, providers=["CPUExecutionProvider"] + ) + rec = ort.InferenceSession( + rec_bytes, sess_options=options, providers=["CPUExecutionProvider"] + ) + det.disable_fallback() + rec.disable_fallback() + except Exception as error: + raise AdapterError("MODEL_VERSION_UNAVAILABLE", "OCR model loading failed") from error + return cls(det, rec, chars) + + @staticmethod + def _run(session, tensor): + try: + outputs = session.run(["fetch_name_0"], {"x": tensor}) + except Exception as error: + raise AdapterError("MODEL_ERROR", "OCR inference failed") from error + if not isinstance(outputs, (list, tuple)) or len(outputs) != 1: + raise AdapterError("SCHEMA_MISMATCH", "Unexpected OCR output count") + return outputs[0] + + def recognize(self, image): + import numpy as np + from PIL import Image + + if not isinstance(image, Image.Image) or image.mode != "RGB": + raise AdapterError("SCHEMA_MISMATCH", "OCR input must be RGB") + rgb = np.asarray(image, dtype=np.uint8) + tensor = det_tensor(rgb) + probs = self._run(self.det, tensor) + if getattr(probs, "shape", None) != (1, 1, *tensor.shape[2:]): + raise AdapterError("SCHEMA_MISMATCH", "DB output spatial shape mismatch") + rows = db_regions(probs, image.size) + source_digest = rgb_digest(rgb) + for row in rows: + crop, row["crop_evidence"] = crop_with_evidence(rgb, row["quad_pixels"]) + row["crop_evidence"]["source_rgb_sha256"] = source_digest + rec_input = rec_tensor(crop) + probs = self._run(self.rec, rec_input) + if getattr(probs, "shape", ()) != (1, rec_input.shape[3] // 8, 18710): + raise AdapterError("SCHEMA_MISMATCH", "CTC output sequence shape mismatch") + row["text"], row["confidence"] = decode_ctc(probs, self.characters) + row["quad"] = ( + row.pop("quad_pixels") + / np.array([max(1, image.width - 1), max(1, image.height - 1)]) + ).tolist() + return rows + + def smoke(self): + import numpy as np + from PIL import Image + + with Image.new("RGB", (96, 64), "white") as image: + self.recognize(image) + probs = self._run(self.rec, np.zeros((1, 3, 48, 320), dtype=np.float32)) + decode_ctc(probs, self.characters) diff --git a/apps/ai_inference/adapters/quality.py b/apps/ai_inference/adapters/quality.py new file mode 100644 index 0000000..2027836 --- /dev/null +++ b/apps/ai_inference/adapters/quality.py @@ -0,0 +1,104 @@ +"""Real RGB quality-rgb-lap1-v1 pixel implementation, independent of models.""" + +import math +from dataclasses import asdict, dataclass +from importlib.metadata import version + +from apps.ai_inference.adapters.dfine import AdapterError + +ALGORITHM_ID = "quality-rgb-lap1-v1" + + +@dataclass(frozen=True) +class QualityThresholds: + blur_min: float = 80.0 + dark_min: float = 0.12 + glare_max: float = 0.30 + + def validate(self): + for value in (self.blur_min, self.dark_min, self.glare_max): + if type(value) not in (int, float) or not math.isfinite(value): + raise AdapterError("VALIDATION_ERROR", "Quality thresholds must be finite numbers") + if self.blur_min < 0 or not 0 <= self.dark_min <= 1 or not 0 <= self.glare_max <= 1: + raise AdapterError("VALIDATION_ERROR", "Quality thresholds are outside their ranges") + + +def resize_shape(width, height): + if type(width) is not int or type(height) is not int or min(width, height) < 1: + raise AdapterError("IMAGE_INVALID", "Invalid quality image dimensions") + longest = max(width, height) + if longest <= 1024: + return width, height + return tuple(max(1, (v * 1024 + longest // 2) // longest) for v in (width, height)) + + +def quality_runtime(): + return { + "algorithm_id": ALGORITHM_ID, + "packages": {name: version(name) for name in ("numpy", "Pillow")}, + "resize": { + "max_side": 1024, + "interpolation": "Pillow-BILINEAR", + "rounding": "integer-half-up", + "upscale": False, + "padding": False, + }, + "gray": {"weights": [77, 150, 29], "offset": 128, "divisor": 256, "dtype": "uint32"}, + "laplacian": { + "kernel": [[0, 1, 0], [1, -4, 1], [0, 1, 0]], + "border": "REFLECT_101", + "singleton_axis": "repeat", + "dtype": "float64", + "variance_ddof": 0, + }, + "brightness_divisor": 255, + "glare_gray_min": 250, + "round_scores": False, + } + + +def evaluate_quality(image, thresholds=QualityThresholds()): + """RGB input is already oriented. Never rotate, enlarge, or convert to BGR.""" + import numpy as np + from PIL import Image + + if not isinstance(image, Image.Image) or image.mode != "RGB": + raise AdapterError("SCHEMA_MISMATCH", "Quality input must be RGB") + if not isinstance(thresholds, QualityThresholds): + raise AdapterError("VALIDATION_ERROR", "Invalid quality configuration") + thresholds.validate() + size = resize_shape(*image.size) + resized = image.resize(size, Image.Resampling.BILINEAR) if size != image.size else image + try: + pixels = np.asarray(resized, dtype=np.uint32) + gray = (77 * pixels[:, :, 0] + 150 * pixels[:, :, 1] + 29 * pixels[:, :, 2] + 128) // 256 + gray = gray.astype(np.uint8) + finally: + if resized is not image: + resized.close() + plane = gray.astype(np.float64) + # numpy reflect repeats the sole value when an axis has length one. + padded = np.pad(plane, 1, mode="reflect") + lap = padded[:-2, 1:-1] + padded[2:, 1:-1] + padded[1:-1, :-2] + padded[1:-1, 2:] - 4 * plane + scores = { + "blur_score": float(lap.var(dtype=np.float64, ddof=0)), + "brightness": float(plane.mean(dtype=np.float64) / 255), + "glare_ratio": float(np.count_nonzero(gray >= 250) / gray.size), + } + reasons = [ + name + for name, failed in ( + ("blur", scores["blur_score"] < thresholds.blur_min), + ("dark", scores["brightness"] < thresholds.dark_min), + ("glare", scores["glare_ratio"] > thresholds.glare_max), + ) + if failed + ] + return { + "algorithm_id": ALGORITHM_ID, + "analysis_size": list(size), + **scores, + "passed": not reasons, + "reasons": reasons, + "thresholds": asdict(thresholds), + } diff --git a/apps/ai_inference/cpu.py b/apps/ai_inference/cpu.py new file mode 100644 index 0000000..3c89627 --- /dev/null +++ b/apps/ai_inference/cpu.py @@ -0,0 +1,243 @@ +"""Bounded local CPU detector. No HTTP image references or business writes.""" + +import hashlib +import io +import json +import math +import multiprocessing +import os +import platform +import sys +import time +import warnings +from dataclasses import dataclass +from importlib.metadata import version +from pathlib import Path +from uuid import UUID, uuid5 + +from apps.ai_inference.adapters.dfine import ADAPTER_ID, MAX_DETECTIONS, AdapterError, OnnxDetector + +MAX_IMAGE_BYTES = 12 * 1024 * 1024 + + +@dataclass(frozen=True) +class CpuOptions: + model_path: str + config_path: str + preprocessor_path: str + detection_min: float = 0.4 + threads: int = 2 + timeout_seconds: float = 180.0 + + def validate(self): + if type(self.detection_min) not in (int, float) or not math.isfinite(self.detection_min): + raise AdapterError("VALIDATION_ERROR", "Invalid detection threshold") + if not 0 <= self.detection_min <= 1: + raise AdapterError("VALIDATION_ERROR", "Invalid detection threshold") + if type(self.threads) is not int or not 1 <= self.threads <= 16: + raise AdapterError("VALIDATION_ERROR", "CPU threads must be 1..16") + if type(self.timeout_seconds) not in (int, float) or not math.isfinite( + self.timeout_seconds + ): + raise AdapterError("VALIDATION_ERROR", "Invalid CPU deadline") + if not 0 < self.timeout_seconds <= 180: + raise AdapterError("VALIDATION_ERROR", "CPU deadline must be within 180 seconds") + + +def _decode_local(path): + from PIL import Image, ImageOps, UnidentifiedImageError + + try: + with Path(path).open("rb") as stream: + raw = stream.read(MAX_IMAGE_BYTES + 1) + except OSError as error: + raise AdapterError("IMAGE_INVALID", "Local image cannot be read") from error + if len(raw) > MAX_IMAGE_BYTES: + raise AdapterError("IMAGE_TOO_LARGE", "Local image exceeds the byte limit") + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(io.BytesIO(raw)) as source: + if source.format not in {"PNG", "JPEG", "WEBP"}: + raise AdapterError("UNSUPPORTED_MEDIA_TYPE", "Unsupported local image format") + if getattr(source, "n_frames", 1) != 1: + raise AdapterError("IMAGE_INVALID", "Animated images are not supported") + width, height = source.size + if max(width, height) > 10000 or width * height > 40_000_000: + raise AdapterError("IMAGE_TOO_LARGE", "Local image dimensions exceed the limit") + source.load() + # The CLI accepts original local images. Worker analysis PNGs have + # already lost EXIF, so this cannot rotate them a second time. + image = ImageOps.exif_transpose(source).convert("RGB") + except AdapterError: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error: + raise AdapterError("IMAGE_TOO_LARGE", "Local image exceeds the pixel limit") from error + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error: + raise AdapterError("IMAGE_INVALID", "Local image is invalid or truncated") from error + return image, hashlib.sha256(raw).hexdigest() + + +def _compute(options, paths, run_id): + """Executed only in a spawned child; load the model once for all images.""" + from PIL import Image + + started = time.monotonic() + detector = OnnxDetector.from_path( + options.model_path, + providers=["CPUExecutionProvider"], + config_path=options.config_path, + preprocessor_path=options.preprocessor_path, + cpu_threads=options.threads, + ) + if detector.providers != ("CPUExecutionProvider",): + raise AdapterError("MODEL_VERSION_UNAVAILABLE", "CPU-only session is required") + smoke = Image.new("RGB", (640, 640)) + try: + detector.detect(smoke, options.detection_min) + finally: + smoke.close() + load_ms = round((time.monotonic() - started) * 1000) + evidence = detector.evidence + runtime = { + "python": platform.python_version(), + "platform": platform.platform(), + "packages": {name: version(name) for name in ("onnxruntime", "numpy", "Pillow")}, + "providers": list(detector.providers), + "intra_op_threads": options.threads, + "inter_op_threads": 1, + "execution_mode": "sequential", + "preprocess": "rgb-stretch640-bilinear-rescale255", + "postprocess": "qmax-sigmoid-threshold-geometry-no-nms", + } + runtime_sha = hashlib.sha256( + json.dumps(runtime, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + ).hexdigest() + images, total = [], 0 + for ordinal, path in enumerate(paths): + image, digest = _decode_local(path) + try: + detected_at = time.monotonic() + diagnostics = {} + rows = detector.detect(image, options.detection_min, diagnostics=diagnostics) + detection_ms = round((time.monotonic() - detected_at) * 1000) + total += len(rows) + if total > MAX_DETECTIONS: + raise AdapterError("MODEL_ERROR", "All images exceed the 100-detection limit") + image_id = str(uuid5(UUID(run_id), f"image:{ordinal}:{digest}")) + detections = [ + { + "detection_id": str(uuid5(UUID(run_id), f"{image_id}:{row['type']}:{index}")), + "image_id": image_id, + "parent_detection_id": None, + **{key: row[key] for key in ("class_id", "type", "bbox", "confidence")}, + } + for index, row in enumerate(rows) + ] + images.append( + { + "image_id": image_id, + "input_sha256": digest, + "width": image.width, + "height": image.height, + "diagnostics": diagnostics, + "detection_ms": detection_ms, + "detections": detections, + } + ) + finally: + image.close() + return { + "schema_version": "coco80-local-v1", + "run_id": run_id, + "purpose": "development", + "is_simulated": False, + "device": "cpu", + "adapter_id": ADAPTER_ID, + "model_sha256": evidence.model_sha256, + "config_sha256": evidence.config_sha256, + "preprocessor_sha256": evidence.preprocessor_sha256, + "runtime": runtime, + "runtime_sha256": runtime_sha, + "detection_min": options.detection_min, + "review_required": True, + "business_capabilities": { + "ocr": False, + "chemical_entities": False, + "container_relations": False, + }, + "load_and_smoke_ms": load_ms, + "total_ms": round((time.monotonic() - started) * 1000), + "images": images, + } + + +def _child(channel, options, paths, run_id): + try: + channel.send(("result", _compute(options, paths, run_id))) + except AdapterError as error: + channel.send(("error", error.code)) + except MemoryError: + channel.send(("error", "MODEL_OOM")) + except Exception: + channel.send(("error", "MODEL_ERROR")) + finally: + channel.close() + + +def run_cpu_detection(options: CpuOptions, paths, run_id: str): + return run_cpu_workload(options, paths, run_id, _child) + + +def run_cpu_workload(options, paths, run_id: str, target): + """One CPU child, one total deadline, no partial result after failure.""" + if sys.version_info[:2] != (3, 11): + raise AdapterError("VALIDATION_ERROR", "CPU runtime requires Python 3.11") + if os.environ.get("APP_ENV") not in {"dev", "test"}: + raise AdapterError("VALIDATION_ERROR", "Local CPU inference requires APP_ENV=dev or test") + options.validate() + try: + run_id = str(UUID(run_id)) + except (ValueError, TypeError, AttributeError) as error: + raise AdapterError("VALIDATION_ERROR", "Invalid run ID") from error + if not isinstance(paths, (list, tuple)) or not 1 <= len(paths) <= 3: + raise AdapterError("VALIDATION_ERROR", "Supply 1..3 local images") + paths = tuple(str(Path(path).resolve()) for path in paths) + context = multiprocessing.get_context("spawn") + receive, send = context.Pipe(duplex=False) + child = context.Process(target=target, args=(send, options, paths, run_id), daemon=True) + deadline = time.monotonic() + options.timeout_seconds + try: + child.start() + send.close() + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise AdapterError("AI_TIMEOUT", "CPU inference deadline exceeded") + if receive.poll(min(remaining, 0.1)): + try: + kind, payload = receive.recv() + except (EOFError, OSError) as error: + raise AdapterError( + "MODEL_ERROR", "CPU child exited without a result" + ) from error + if time.monotonic() >= deadline: + raise AdapterError("AI_TIMEOUT", "CPU inference deadline exceeded") + if kind == "result" and isinstance(payload, dict): + return payload + if kind == "error" and isinstance(payload, str): + raise AdapterError(payload, "CPU inference failed") + raise AdapterError("MODEL_ERROR", "Invalid CPU child response") + if not child.is_alive(): + raise AdapterError("MODEL_ERROR", "CPU child exited without a result") + finally: + send.close() + receive.close() + if child.pid is not None: + child.join(timeout=0.2) + if child.is_alive(): + child.kill() + child.join(timeout=2) + if child.is_alive(): + raise AdapterError("AI_TIMEOUT", "CPU child could not be reclaimed") + child.close() diff --git a/apps/ai_inference/detect.py b/apps/ai_inference/detect.py new file mode 100644 index 0000000..d68c9f6 --- /dev/null +++ b/apps/ai_inference/detect.py @@ -0,0 +1,57 @@ +"""CLI for local, real COCO-80 CPU detection, outside the fixture HTTP service.""" + +import argparse +import json +import sys +from pathlib import Path +from uuid import uuid4 + +from apps.ai_inference.adapters.dfine import AdapterError +from apps.ai_inference.cpu import CpuOptions, run_cpu_detection + + +def main(): + parser = argparse.ArgumentParser(description="Run official COCO-80 detection on CPU") + parser.add_argument("--model", required=True, type=Path) + parser.add_argument("--config", type=Path) + parser.add_argument("--preprocessor", type=Path) + parser.add_argument("--input", required=True, nargs="+", type=Path) + parser.add_argument("--threshold", type=float, default=0.4) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--run-id", default=str(uuid4())) + parser.add_argument("--output", type=Path, help="Write JSON after all images succeed") + args = parser.parse_args() + options = CpuOptions( + str(args.model.resolve()), + str((args.config or args.model.with_name("config.json")).resolve()), + str((args.preprocessor or args.model.with_name("preprocessor_config.json")).resolve()), + args.threshold, + args.threads, + args.timeout, + ) + try: + result = run_cpu_detection(options, args.input, args.run_id) + encoded = json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False) + "\n" + if args.output is None: + print(encoded, end="") + else: + # Exclusive create prevents accidentally replacing a previous report. + with args.output.open("x", encoding="utf-8") as stream: + stream.write(encoded) + except AdapterError as error: + print(json.dumps({"error": {"code": error.code, "message": str(error)}}), file=sys.stderr) + return 1 + except OSError: + print( + json.dumps( + {"error": {"code": "OUTPUT_UNAVAILABLE", "message": "Cannot create output file"}} + ), + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/ai_inference/ocr.py b/apps/ai_inference/ocr.py new file mode 100644 index 0000000..38710e6 --- /dev/null +++ b/apps/ai_inference/ocr.py @@ -0,0 +1,51 @@ +"""Real CPU text detection and recognition using locally supplied ONNX files.""" + +import argparse +import json +import sys +from pathlib import Path +from uuid import uuid4 + +from apps.ai_inference.adapters.dfine import AdapterError +from apps.ai_inference.ocr_cpu import OcrOptions, run_cpu_ocr + + +def main(): + parser = argparse.ArgumentParser(description="PP-OCRv6_small local CPU OCR") + parser.add_argument("--det-dir", type=Path, required=True) + parser.add_argument("--rec-dir", type=Path, required=True) + parser.add_argument("--input", type=Path, required=True, nargs="+") + parser.add_argument("--text-min", type=float, default=0.6) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--run-id", default=str(uuid4())) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + options = OcrOptions( + str(args.det_dir.resolve()), + str(args.rec_dir.resolve()), + args.text_min, + args.threads, + args.timeout, + ) + try: + result = run_cpu_ocr(options, args.input, args.run_id) + text = json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False) + "\n" + if args.output is None: + # Handle non-BMP dictionary entries on Windows consoles. + sys.stdout.reconfigure(encoding="utf-8") + print(text, end="") + else: + with args.output.open("x", encoding="utf-8") as stream: + stream.write(text) + except AdapterError as error: + print(json.dumps({"error": {"code": error.code, "message": str(error)}}), file=sys.stderr) + return 1 + except OSError: + print(json.dumps({"error": {"code": "OUTPUT_UNAVAILABLE"}}), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/ai_inference/ocr_cpu.py b/apps/ai_inference/ocr_cpu.py new file mode 100644 index 0000000..bcc76eb --- /dev/null +++ b/apps/ai_inference/ocr_cpu.py @@ -0,0 +1,142 @@ +"""Bounded local OCR report, independent of chemical facts and the HTTP fixture.""" + +import hashlib +import json +import platform +import time +from dataclasses import dataclass +from importlib.metadata import version +from uuid import UUID, uuid5 + +from apps.ai_inference.adapters.dfine import AdapterError +from apps.ai_inference.adapters.ocrv6 import ADAPTER_ID, ARTIFACTS, MAX_REGIONS, OnnxOCR +from apps.ai_inference.cpu import CpuOptions, _decode_local, run_cpu_workload +from packages.image_evidence.perspective import crop_runtime + + +@dataclass(frozen=True) +class OcrOptions: + det_dir: str + rec_dir: str + text_min: float = 0.6 + threads: int = 2 + timeout_seconds: float = 180.0 + + def validate(self): + CpuOptions("", "", "", self.text_min, self.threads, self.timeout_seconds).validate() + + +def _compute(options, paths, run_id): + started = time.monotonic() + ocr = OnnxOCR.from_directories(options.det_dir, options.rec_dir, options.threads) + ocr.smoke() + loaded_ms = round((time.monotonic() - started) * 1000) + images, total = [], 0 + for ordinal, path in enumerate(paths): + image, digest = _decode_local(path) + try: + inferred_at = time.monotonic() + rows = ocr.recognize(image) + total += len(rows) + if total > MAX_REGIONS: + raise AdapterError("MODEL_ERROR", "All images exceed the OCR region capacity") + image_id = str(uuid5(UUID(run_id), f"image:{ordinal}:{digest}")) + annotate_lines(rows, run_id, image_id, options.text_min) + images.append( + { + "image_id": image_id, + "input_sha256": digest, + "width": image.width, + "height": image.height, + "ocr_ms": round((time.monotonic() - inferred_at) * 1000), + "lines": rows, + } + ) + finally: + image.close() + runtime = ocr_runtime(options.threads) + return { + "schema_version": "ocrv6-local-v2", + "run_id": run_id, + "purpose": "development", + "is_simulated": False, + "device": "cpu", + "adapter_id": ADAPTER_ID, + "artifacts": ARTIFACTS, + "runtime": runtime, + "runtime_sha256": hashlib.sha256( + json.dumps(runtime, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + ).hexdigest(), + "text_min": options.text_min, + "review_required": True, + "business_capabilities": { + "text_recognition": True, + "chemical_entities": False, + "container_relations": False, + "bottle_association": False, + }, + "load_and_smoke_ms": loaded_ms, + "total_ms": round((time.monotonic() - started) * 1000), + "images": images, + } + + +def annotate_lines(rows, run_id, image_id, text_min): + for index, row in enumerate(rows): + row.update( + line_id=str(uuid5(UUID(run_id), f"ocr:{image_id}:{index}")), + image_id=image_id, + parent_detection_id=None, + uncertain=not row["text"] or row["confidence"] < text_min, + ) + row["crop_evidence"].update( + crop_id=str(uuid5(UUID(run_id), f"ocr-crop:{image_id}:{index}")), + image_id=image_id, + line_id=row["line_id"], + ) + + +def ocr_runtime(threads): + runtime = { + "python": platform.python_version(), + "platform": platform.platform(), + "packages": { + name: version(name) + for name in ( + "onnxruntime", + "numpy", + "Pillow", + "opencv-python-headless", + "pyclipper", + "PyYAML", + ) + }, + "providers": ["CPUExecutionProvider"], + "intra_op_threads": threads, + "inter_op_threads": 1, + "execution_mode": "sequential", + "det_preprocess": "bgr-imagenet-max960-round32-v1", + "det_postprocess": "db-thresh0.2-box0.45-unclip1.4-no-dilation-v1", + "rec_preprocess": "bgr-height48-minwidth320-align8-max3200-pad0-v1", + "rec_postprocess": "ctc-blank0-deduplicate-append-space-v1", + "crop": crop_runtime(), + "orientation_classifier": False, + } + return runtime + + +def _child(channel, options, paths, run_id): + try: + channel.send(("result", _compute(options, paths, run_id))) + except AdapterError as error: + channel.send(("error", error.code)) + except MemoryError: + channel.send(("error", "MODEL_OOM")) + except Exception: + channel.send(("error", "MODEL_ERROR")) + finally: + channel.close() + + +def run_cpu_ocr(options, paths, run_id): + return run_cpu_workload(options, paths, run_id, _child) diff --git a/apps/ai_inference/pipeline.py b/apps/ai_inference/pipeline.py new file mode 100644 index 0000000..58e2113 --- /dev/null +++ b/apps/ai_inference/pipeline.py @@ -0,0 +1,65 @@ +"""CLI for real quality gating followed by joint local CPU detection/OCR.""" + +import argparse +import json +import sys +from pathlib import Path +from uuid import uuid4 + +from apps.ai_inference.adapters.dfine import AdapterError +from apps.ai_inference.adapters.quality import QualityThresholds +from apps.ai_inference.pipeline_cpu import PipelineOptions, run_cpu_pipeline + + +def main(): + parser = argparse.ArgumentParser(description="Quality + COCO80 + PP-OCRv6_small CPU pipeline") + parser.add_argument("--model", type=Path) + parser.add_argument("--config", type=Path) + parser.add_argument("--preprocessor", type=Path) + parser.add_argument("--det-dir", type=Path) + parser.add_argument("--rec-dir", type=Path) + parser.add_argument("--quality-only", action="store_true") + parser.add_argument("--input", type=Path, required=True, nargs="+") + parser.add_argument("--threshold", type=float, default=0.4) + parser.add_argument("--text-min", type=float, default=0.6) + parser.add_argument("--blur-min", type=float, default=80) + parser.add_argument("--dark-min", type=float, default=0.12) + parser.add_argument("--glare-max", type=float, default=0.30) + parser.add_argument("--threads", type=int, default=2) + parser.add_argument("--timeout", type=float, default=180) + parser.add_argument("--run-id", default=str(uuid4())) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + options = PipelineOptions( + model_path=str(args.model.resolve()) if args.model else None, + det_dir=str(args.det_dir.resolve()) if args.det_dir else None, + rec_dir=str(args.rec_dir.resolve()) if args.rec_dir else None, + config_path=str(args.config.resolve()) if args.config else None, + preprocessor_path=str(args.preprocessor.resolve()) if args.preprocessor else None, + detection_min=args.threshold, + text_min=args.text_min, + quality=QualityThresholds(args.blur_min, args.dark_min, args.glare_max), + quality_only=args.quality_only, + threads=args.threads, + timeout_seconds=args.timeout, + ) + try: + result = run_cpu_pipeline(options, args.input, args.run_id) + encoded = json.dumps(result, ensure_ascii=False, indent=2, allow_nan=False) + "\n" + if args.output is None: + sys.stdout.reconfigure(encoding="utf-8") + print(encoded, end="") + else: + with args.output.open("x", encoding="utf-8") as stream: + stream.write(encoded) + except AdapterError as error: + print(json.dumps({"error": {"code": error.code, "message": str(error)}}), file=sys.stderr) + return 1 + except OSError: + print(json.dumps({"error": {"code": "OUTPUT_UNAVAILABLE"}}), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/ai_inference/pipeline_cpu.py b/apps/ai_inference/pipeline_cpu.py new file mode 100644 index 0000000..280d287 --- /dev/null +++ b/apps/ai_inference/pipeline_cpu.py @@ -0,0 +1,209 @@ +"""Local quality gate and joint COCO-80/OCR execution in one bounded CPU child.""" + +import hashlib +import json +import platform +import time +from dataclasses import asdict, dataclass +from uuid import UUID, uuid5 + +from apps.ai_inference.adapters.dfine import ADAPTER_ID, MAX_DETECTIONS, AdapterError, OnnxDetector +from apps.ai_inference.adapters.ocrv6 import ARTIFACTS, MAX_REGIONS, OnnxOCR +from apps.ai_inference.adapters.quality import QualityThresholds, evaluate_quality, quality_runtime +from apps.ai_inference.cpu import CpuOptions, _decode_local, run_cpu_workload +from apps.ai_inference.ocr_cpu import OcrOptions, annotate_lines, ocr_runtime + +PIPELINE_ID = "quality-coco80-ocrv6-cpu-v2" + + +@dataclass(frozen=True) +class PipelineOptions: + model_path: str | None = None + det_dir: str | None = None + rec_dir: str | None = None + config_path: str | None = None + preprocessor_path: str | None = None + detection_min: float = 0.4 + text_min: float = 0.6 + quality: QualityThresholds = QualityThresholds() + quality_only: bool = False + threads: int = 2 + timeout_seconds: float = 180.0 + + def validate(self): + CpuOptions("", "", "", self.detection_min, self.threads, self.timeout_seconds).validate() + OcrOptions("", "", self.text_min, self.threads, self.timeout_seconds).validate() + if type(self.quality_only) is not bool or not isinstance(self.quality, QualityThresholds): + raise AdapterError("VALIDATION_ERROR", "Invalid pipeline configuration") + self.quality.validate() + if not self.quality_only and any( + not isinstance(p, str) or not p for p in (self.model_path, self.det_dir, self.rec_dir) + ): + raise AdapterError("VALIDATION_ERROR", "Detector and OCR paths are required") + + +def _digest(value): + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + ).hexdigest() + + +def _compute(options, paths, run_id): + """Decode once, gate the whole batch, then load each session once.""" + from PIL import Image + + started = time.monotonic() + decoded, images = [], [] + quality_ms = 0 + try: + for ordinal, path in enumerate(paths): + image, digest = _decode_local(path) + decoded.append(image) + checked_at = time.monotonic() + quality = evaluate_quality(image, options.quality) + quality_ms += round((time.monotonic() - checked_at) * 1000) + image_id = str(uuid5(UUID(run_id), f"image:{ordinal}:{digest}")) + images.append( + { + "image_id": image_id, + "input_sha256": digest, + "width": image.width, + "height": image.height, + "quality": quality, + "inference_executed": False, + "detections": [], + "lines": [], + "detection_ms": 0, + "ocr_ms": 0, + } + ) + passed = all(row["quality"]["passed"] for row in images) + runtime = { + "python": platform.python_version(), + "platform": platform.platform(), + "quality": quality_runtime(), + "detector": None, + "ocr": None, + } + artifacts, loaded_ms = {}, 0 + models_executed = passed and not options.quality_only + if models_executed: + loaded_at = time.monotonic() + detector = OnnxDetector.from_path( + options.model_path, + providers=["CPUExecutionProvider"], + config_path=options.config_path, + preprocessor_path=options.preprocessor_path, + cpu_threads=options.threads, + ) + if detector.providers != ("CPUExecutionProvider",): + raise AdapterError("MODEL_VERSION_UNAVAILABLE", "CPU-only detector required") + with Image.new("RGB", (640, 640)) as smoke: + detector.detect(smoke, options.detection_min) + ocr = OnnxOCR.from_directories(options.det_dir, options.rec_dir, options.threads) + ocr.smoke() + loaded_ms = round((time.monotonic() - loaded_at) * 1000) + evidence = detector.evidence + artifacts = { + "detector": { + "model_sha256": evidence.model_sha256, + "config_sha256": evidence.config_sha256, + "preprocessor_sha256": evidence.preprocessor_sha256, + }, + "ocr": ARTIFACTS, + } + runtime["detector"] = { + "adapter_id": ADAPTER_ID, + "providers": list(detector.providers), + "intra_op_threads": options.threads, + "inter_op_threads": 1, + "execution_mode": "sequential", + "preprocess": "rgb-stretch640-bilinear-rescale255", + "postprocess": "qmax-sigmoid-threshold-geometry-no-nms", + } + runtime["ocr"] = ocr_runtime(options.threads) + detections_total, lines_total = 0, 0 + for image, row in zip(decoded, images): + inferred_at = time.monotonic() + diagnostics = {} + detections = detector.detect(image, options.detection_min, diagnostics=diagnostics) + row["detection_ms"] = round((time.monotonic() - inferred_at) * 1000) + detections_total += len(detections) + if detections_total > MAX_DETECTIONS: + raise AdapterError("MODEL_ERROR", "Pipeline detection capacity exceeded") + inferred_at = time.monotonic() + lines = ocr.recognize(image) + row["ocr_ms"] = round((time.monotonic() - inferred_at) * 1000) + lines_total += len(lines) + if lines_total > MAX_REGIONS: + raise AdapterError("MODEL_ERROR", "Pipeline OCR region capacity exceeded") + row["detections"] = [ + { + "detection_id": str( + uuid5(UUID(run_id), f"{row['image_id']}:{d['type']}:{index}") + ), + "image_id": row["image_id"], + "parent_detection_id": None, + **{key: d[key] for key in ("class_id", "type", "bbox", "confidence")}, + } + for index, d in enumerate(detections) + ] + annotate_lines(lines, run_id, row["image_id"], options.text_min) + row.update(inference_executed=True, lines=lines, detection_diagnostics=diagnostics) + config = { + "pipeline_id": PIPELINE_ID, + "detection_min": options.detection_min, + "text_min": options.text_min, + "quality": asdict(options.quality), + "quality_only": options.quality_only, + } + return { + "schema_version": "cpu-pipeline-local-v2", + "pipeline_id": PIPELINE_ID, + "run_id": run_id, + "purpose": "development", + "is_simulated": False, + "device": "cpu", + "outcome": "needs_retake" + if not passed + else ("quality_passed" if options.quality_only else "needs_review"), + "review_required": True, + "models_executed": models_executed, + "artifacts": artifacts, + "runtime": runtime, + "runtime_sha256": _digest(runtime), + "config": config, + "config_sha256": _digest(config), + "business_capabilities": { + "chemical_entities": False, + "container_relations": False, + "bottle_association": False, + "date_facts": False, + }, + "timing_ms": { + "quality_ms": quality_ms, + "load_and_smoke_ms": loaded_ms, + "total_ms": round((time.monotonic() - started) * 1000), + }, + "images": images, + } + finally: + for image in decoded: + image.close() + + +def _child(channel, options, paths, run_id): + try: + channel.send(("result", _compute(options, paths, run_id))) + except AdapterError as error: + channel.send(("error", error.code)) + except MemoryError: + channel.send(("error", "MODEL_OOM")) + except Exception: + channel.send(("error", "MODEL_ERROR")) + finally: + channel.close() + + +def run_cpu_pipeline(options, paths, run_id): + return run_cpu_workload(options, paths, run_id, _child) diff --git a/contracts/development-acceptance-policy.json b/contracts/development-acceptance-policy.json index 7273202..bb1952e 100644 --- a/contracts/development-acceptance-policy.json +++ b/contracts/development-acceptance-policy.json @@ -7,10 +7,7 @@ { "metric_id": "DET-P", "groups": [ - "bottle", - "label", - "shelf", - "cabinet" + "coco80" ], "unit": "ratio", "direction": "ge", @@ -23,10 +20,7 @@ { "metric_id": "DET-R", "groups": [ - "bottle", - "label", - "shelf", - "cabinet" + "coco80" ], "unit": "ratio", "direction": "ge", diff --git a/contracts/inference-v1.yaml b/contracts/inference-v1.yaml index aeb947a..223be0c 100644 --- a/contracts/inference-v1.yaml +++ b/contracts/inference-v1.yaml @@ -633,13 +633,93 @@ components: anyOf: - *id001 - type: 'null' + class_id: + type: integer + minimum: 0 + maximum: 79 type: type: string enum: + - person + - bicycle + - car + - motorbike + - aeroplane + - bus + - train + - truck + - boat + - traffic light + - fire hydrant + - stop sign + - parking meter + - bench + - bird + - cat + - dog + - horse + - sheep + - cow + - elephant + - bear + - zebra + - giraffe + - backpack + - umbrella + - handbag + - tie + - suitcase + - frisbee + - skis + - snowboard + - sports ball + - kite + - baseball bat + - baseball glove + - skateboard + - surfboard + - tennis racket - bottle - - label - - shelf - - cabinet + - wine glass + - cup + - fork + - knife + - spoon + - bowl + - banana + - apple + - sandwich + - orange + - broccoli + - carrot + - hot dog + - pizza + - donut + - cake + - chair + - sofa + - pottedplant + - bed + - diningtable + - toilet + - tvmonitor + - laptop + - mouse + - remote + - keyboard + - cell phone + - microwave + - oven + - toaster + - sink + - refrigerator + - book + - clock + - vase + - scissors + - teddy bear + - hair drier + - toothbrush bbox: type: array items: *id003 @@ -650,6 +730,7 @@ components: - detection_id - image_id - parent_detection_id + - class_id - type - bbox - confidence @@ -970,13 +1051,13 @@ components: adapter_id: type: string enum: - - dfine-n4-rgb-stretch-v1 + - dfine-coco80-rgb-stretch-v1 - fixture-v1 runtime_profile: type: string enum: - - dfine-cpu-fp32-ocrv4cpu-v1 - - dfine-cuda-fp32-ocrv4cpu-v1 + - dfine-cpu-fp32-ocrv6smallcpu-v1 + - dfine-cuda-fp32-ocrv6smallcpu-v1 - fixture-v1 runtime_lock_sha256: type: string diff --git a/contracts/model-manifest-v1.json b/contracts/model-manifest-v1.json index 75928f4..cdfe6f8 100644 --- a/contracts/model-manifest-v1.json +++ b/contracts/model-manifest-v1.json @@ -75,7 +75,7 @@ "ocr_backend": { "type": "string", "enum": [ - "paddleocr", + "onnxruntime", "mock" ] }, @@ -105,7 +105,7 @@ "adapter_id": { "type": "string", "enum": [ - "dfine-n4-rgb-stretch-v1", + "dfine-coco80-rgb-stretch-v1", "fixture-v1" ] }, @@ -128,8 +128,8 @@ "runtime_profile": { "type": "string", "enum": [ - "dfine-cpu-fp32-ocrv4cpu-v1", - "dfine-cuda-fp32-ocrv4cpu-v1", + "dfine-cpu-fp32-ocrv6smallcpu-v1", + "dfine-cuda-fp32-ocrv6smallcpu-v1", "fixture-v1" ] }, diff --git a/contracts/public-api-v1.yaml b/contracts/public-api-v1.yaml index 7751dde..db52569 100644 --- a/contracts/public-api-v1.yaml +++ b/contracts/public-api-v1.yaml @@ -11973,7 +11973,7 @@ components: ocr_backend: type: string enum: - - paddleocr + - onnxruntime - mock evaluation_report_key: anyOf: @@ -11991,7 +11991,7 @@ components: adapter_id: type: string enum: - - dfine-n4-rgb-stretch-v1 + - dfine-coco80-rgb-stretch-v1 - fixture-v1 model_family: type: string @@ -12008,8 +12008,8 @@ components: runtime_profile: type: string enum: - - dfine-cpu-fp32-ocrv4cpu-v1 - - dfine-cuda-fp32-ocrv4cpu-v1 + - dfine-cpu-fp32-ocrv6smallcpu-v1 + - dfine-cuda-fp32-ocrv6smallcpu-v1 - fixture-v1 runtime: $ref: '#/components/schemas/InferenceRuntime' diff --git a/docs/00-document-control.md b/docs/00-document-control.md index 4c07a1b..681f28b 100644 --- a/docs/00-document-control.md +++ b/docs/00-document-control.md @@ -16,8 +16,12 @@ ## 2. 状态与审批 +2026-10-07 用户选择PP-OCRv6_small ONNX并提供det/rec资产，替代原PP-OCRv4/Paddle CPU选型；当前CPU、不安装CUDA。ocr_backend及runtime_profile生成契约同步变更，旧v4/paddleocr组合不兼容；不能给历史制品或报告改名复用。文字本地实现与完整业务接线分别验收，见[OCR续批](08-delivery/33-i-ml01-ocrv6-cpu.md)。 + “结构校验通过”“设计可编码”“协作者批准”“系统可上线”分别判断。无签署记录不得称已批准。实际模型、依赖兼容性、数据许可、专家规则和性能成绩不可虚构。 +2026-10-07 [可重建OCR证据续批](08-delivery/35-i-ml01-rebuildable-ocr-evidence.md)统一真实OCR裁剪到perspective-rgb-v1黑色边界，替代本地旧复制边缘算法；适配器和本地报告升级v2，历史33/34证据保留。RPC/业务引用闭包未改变，不把局部证据基础记作已落库或已批准。 + ## 3. 排版和维护 主题目录两位数字排序；README 只做导航；正文一级标题为文档、二级为职责、三级为规则。业务变更同步更新主题文档、契约、测试用例、追踪矩阵与审计。所有现行链接必须指向存在的文件。 diff --git a/docs/02-architecture/03-ai-inference-process.md b/docs/02-architecture/03-ai-inference-process.md index 60cf88a..b47dd05 100644 --- a/docs/02-architecture/03-ai-inference-process.md +++ b/docs/02-architecture/03-ai-inference-process.md @@ -10,12 +10,16 @@ AI 是单 Uvicorn worker supervisor + 一个 multiprocessing spawn 计算子进 AI 只能读授权 analysis/ 对象和只读模型目录，无 MySQL/Redis/业务 ORM/规则依赖。Worker 根据裁剪配方写证据对象。AI 不返回可写 URL、不持有写权限。 -模型和数据路线以[固定模型方案](../04-ai-rules/02-model-data-plan.md)为准：AI运行时Python3.11、D-FINE-N四类ONNX，业务同为Python3.11但保留独立依赖环境；RGB直接resize640并除255，qmax/无NMS，不套用其他模型的letterbox。建议生产检测CUDA FP32；OCR保持PP-OCRv4 CPU。APP_ENV/AI_MODE在启动校验；production禁止fixture和未批准bundle，dev/test可用同协议mock进程。/version除purpose/is_simulated外回显adapter_id、runtime_profile、runtime_lock_sha256、detector_device和ocr_device；这些值取实际通过启动校验的加载状态，Worker与受控manifest逐项核对。 +模型和数据路线以[固定模型方案](../04-ai-rules/02-model-data-plan.md)及本批验收为准：AI运行时 Python3.11、官方 D-FINE COCO 80 类 ONNX，业务同为 Python3.11 但保留独立依赖环境；RGB直接resize640并除255，官方输出为 300 queries、qmax/无NMS，协议保留 0–79 `class_id` 和对应 `type`。建议生产检测 CUDA FP32；OCR保持 PP-OCRv6_small ONNX CPU。APP_ENV/AI_MODE 在启动校验；production 禁止 fixture 和未批准 bundle，dev/test 可用同协议 mock 进程。/version 除 purpose/is_simulated 外回显 adapter_id、runtime_profile、runtime_lock_sha256、detector_device 和 ocr_device；这些值取实际通过启动校验的加载状态，Worker 与受控 manifest 逐项核对。 -I-01A只交付同协议开发fixture和HTTP进程入口；下面的真实spawn计算子进程、模型加载与故障恢复仍按I-03/I-ML实施，不能将fixture冒充完整supervisor。运行和验收入口见[开发指南](../../DEVELOPMENT.md)。 +I-01A只交付同协议开发fixture和HTTP进程入口；I-ML-01 [CPU 本地检测续批](../08-delivery/32-i-ml01-cpu-detection.md) 已实现一次性spawn、真实模型加载/smoke、1–3图检测和超时回收。当前仅使用CPU，不安装CUDA；HTTP仍为AI_MODE=mock。本地coco80-local-v1报告尚未接受控analysis读取、quality、Worker RPC或事实落库；下面的常驻supervisor、完整流水线与版本激活仍待I-03/I-ML接通。运行和验收入口见[开发指南](../../DEVELOPMENT.md)。 ## 2. 生命周期 +[联合CPU续批](../08-delivery/34-i-ml01-quality-cpu-pipeline.md)已实现本地真实质量门禁和同一子进程的检测/OCR顺序执行。全部原图先检查质量，任一失败返回整批needs_retake、跳过模型；quality-only与联合入口使用同一函数。仍未接受控analysis/HTTP，且不符合完整InferenceResult；下述常驻服务生命周期及RPC阶段预算未由本地180秒总期限替代。 + +2026-10-07 [PP-OCRv6_small CPU续批](../08-delivery/33-i-ml01-ocrv6-cpu.md) 已接独立本地文字检测/识别，原图文字区域不依赖COCO标签类别。当前ocrv6-local-v1行报告保留原文/置信度/quad、无瓶子归属或化学字段；不是完整InferenceResult。后续业务接线须明确OCR文字区域如何纳入证据闭包，不能借用任意COCO类别填充label/detection/crop引用。以下完整流水线中的label关联仍待落实。 + ### 静态路由与多版本 Worker加载受控AI_ROUTES_FILE，其结构见[路由Schema](../../contracts/inference-routes-v1.json)。键(tenant_id,model_bundle_id,dictionary_version_id,device_profile)必须唯一，对应一个管理员预先部署的AI endpoint及token_file。endpoint只能来自配置中的内网allowlist，不从业务请求接受URL。每实例只加载一个固定bundle；不做请求级切换或跨主机调度。初始部署可仅一条route；不同租户/版本需要独立实例和经过容量审核的资源，不能在同一GPU上盲目复制模型。 @@ -81,6 +85,8 @@ HTTP connect=2秒、池等待=1秒、读写取剩余总预算；不能仅靠 soc ## 7. 结果落库与恢复 +[第35批可重建OCR证据](../08-delivery/35-i-ml01-rebuildable-ocr-evidence.md)已实现packages/image_evidence/perspective.py并接真实OCR本地入口。AI/Worker共用配方重建函数，PNG固定RGB/Pillow compress9、无元数据；记录源RGB/PNG/识别像素摘要。识别的90°旋转单独记录，证据PNG保留原图方向。运行记录含图像依赖版本、OpenCV build摘要、zlib版本，不把同版本号当跨平台字节一致性保证，重建时实际核SHA。独立/联合本地报告升级v2；当前OCR证据属于独立文字区域，不冒充RPC强制detection_id的CropRecipe，Worker写证据与下述提交尚未接通。 + 先校验 Schema，再核 tenant/run/attempt/token、输入 hash、model/dictionary/pipeline；所有证据 ID 必须在本结果闭包内；父子图、坐标和 relation 归属再做语义检查。不合法为 SCHEMA_MISMATCH，禁止自动补零或改字段。 锁 item/task_run；lease_owner、token、未过期 lease、当前 run/submission_revision 全部相等才接纳。旧结果 LEASE_LOST，不更新任何当前指针。规则评估后 item=needs_review；零候选也不自动完成。见 [持久任务](05-durable-jobs.md)。 diff --git a/docs/02-architecture/04-coding-baseline.md b/docs/02-architecture/04-coding-baseline.md index f233a54..3c03969 100644 --- a/docs/02-architecture/04-coding-baseline.md +++ b/docs/02-architecture/04-coding-baseline.md @@ -13,7 +13,7 @@ | apps/web | Vue 3、TypeScript、Vite、Element Plus、Vue Router、TanStack Query；Pinia 只管会话/UI | 复制状态机、长期对象凭据 | | apps/api | Python 3.11、FastAPI、Pydantic v2；路由调用命令/查询 | 加载模型，外部调用持有 DB 事务 | | apps/worker | Celery、Redis、持久任务、AI client、规则、裁剪、导出 | 覆盖历史结果 | -| apps/ai_inference | Python3.11独立FastAPI supervisor + spawn计算子进程；D-FINE-N四类ONNX/PP-OCRv4 CPU；建议检测CUDA FP32，CPU功能保留；业务API/Worker同为Python3.11，依赖环境仍隔离 | 业务ORM、数据库、Redis、最终风险 | +| apps/ai_inference | Python3.11独立FastAPI supervisor + spawn计算子进程；官方 D-FINE-N COCO 80 类 ONNX/PP-OCRv6_small ONNX CPU；建议检测CUDA FP32，CPU功能保留；业务API/Worker同为Python3.11，依赖环境仍隔离 | 业务ORM、数据库、Redis、最终风险 | | packages/domain | 状态、guards、权限动作、不变事实类型 | 通用 status setter | | packages/application | 命令处理器、查询、unit_of_work；每命令唯一入口 | 路由另写业务逻辑 | | packages/persistence | SQLAlchemy 2、MySQL 8.0.16+、Alembic、Outbox repository | 省略租户条件 | diff --git a/docs/02-architecture/07-python-runtime.md b/docs/02-architecture/07-python-runtime.md index d8731e8..58f9d87 100644 --- a/docs/02-architecture/07-python-runtime.md +++ b/docs/02-architecture/07-python-runtime.md @@ -26,7 +26,7 @@ | AI | requirements/py311-ai.txt；I-01A 无 Celery、Redis、业务 ORM 或真实模型包 | | 共用部分 | common.txt 固定当前直接依赖；inference_protocol 只包含 wire schema、DTO 校验和规范哈希 | | Python 版本 | pyproject 要求 >=3.11,<3.12；运行时显式校验 minor；CI 使用 3.11 | -| 模型 | D-FINE-N、检测 CUDA FP32 建议、PP-OCRv4 CPU 及真实 CPU profile 均不改变 | +| 模型 | D-FINE-N、检测 CUDA FP32 建议、PP-OCRv6_small ONNX CPU 及真实 CPU profile 均不改变 | | 环境安全 | I-01A 仅 dev/test，AI_MODE=mock；生产与 real 启动拒绝 | FastAPI/Celery 是必需运行依赖，缺失时必须失败，不在应用入口构造兼容外观的假框架。 diff --git a/docs/04-ai-rules/01-data-and-rules.md b/docs/04-ai-rules/01-data-and-rules.md index ef8455a..00d7876 100644 --- a/docs/04-ai-rules/01-data-and-rules.md +++ b/docs/04-ai-rules/01-data-and-rules.md @@ -4,21 +4,23 @@ 模型输出是观测；FactRevision是可人工修订的解释；Finding是确定规则产生、等待人工确认的候选。模型置信度不是安全严重度。不得从OCR未识别推断安全，也不得从模型结果直接派发整改。 -[manifest Schema](../../contracts/model-manifest-v1.json)提供制品、模式、适配器、词典、阈值、校准和评测引用。production必须使用已校准且获得批准的manifest；dev/test允许[模型方案](02-model-data-plan.md)明确标注的联调初值，缺值仍拒绝加载，不临时猜测。bundle有detector/ocr/quality/dictionary各一份、每个SHA逐文件验证。evaluation_passed只是声明，正式validate/publish/activate须按[验收规范](../07-quality-operations/04-acceptance-policy.md)核验真实报告及服务端批准台账。选型和四类项目数据路线已经固定，实测和业务批准属于I/R阶段，不是当前设计必须先产出的结果。 +[manifest Schema](../../contracts/model-manifest-v1.json)提供制品、模式、适配器、词典、阈值、校准和评测引用。production必须使用已校准且获得批准的manifest；dev/test允许[模型方案](02-model-data-plan.md)明确标注的联调初值，缺值仍拒绝加载，不临时猜测。bundle有detector/ocr/quality/dictionary各一份、每个SHA逐文件验证。evaluation_passed只是声明，正式validate/publish/activate须按[验收规范](../07-quality-operations/04-acceptance-policy.md)核验真实报告及服务端批准台账。选型和 COCO 80 类模型路线已经固定，实测和业务批准属于I/R阶段，不是当前设计必须先产出的结果。 ## 2. vision-v1 适配器接口 | 接口 | 输入 | 输出与算法要求 | |---|---|---| | Quality.evaluate | RGB8 H×W×3 | quality-rgb-lap1-v1：最长边仅缩小到1024、PIL BILINEAR；固定整数灰度和四邻域Laplacian总体方差；精确参数见事实提取规范 | -| Detector.detect | RGB8直接resize640/除255，NCHW float32；orig_target_sizes=int64[[W,H]] | dfine-n4-rgb-stretch-v1：labels/boxes/scores各300query；每query仅最大logit类别的sigmoid分数、无NMS；0 bottle/1 label/2 shelf/3 cabinet；原图框裁边归一化，全请求最多100结果，详见模型方案 | -| OCR.recognize | RGB8 label crop | 适配PaddleOCR到有序文本行(text,confidence,quad)，按y再x排序；不得把模型库原始随版本变化的结构透传API | +| Detector.detect | RGB8直接resize640/除255，pixel_values=NCHW float32 | dfine-coco80-rgb-stretch-v1：实际logits[1,300,80]/pred_boxes[1,300,4]，qmax/sigmoid及cxcywh转换；官方 COCO 0–79 类、无四类压缩、无NMS；保留框裁边归一化，全请求最多100结果，详见模型方案 | +| OCR.recognize | RGB8原图或明确文字区域；适配层转BGR | PP-OCRv6_small ONNX CPU：DB文字检测、透视裁剪、CTC解码到有序文本行(text,confidence,quad)，按y再x排序；不透传模型原始张量 | | Dictionary.resolve | OCR字段、冻结词典 | 规范化NFKC+casefold、空白压缩；优先CAS精确，再alias精确，再规范Levenshtein相似度1-distance/maxlen；降序score再entity_id，top5 | | Relations.evaluate | 同图overview的bottle boxes及容器归属 | 中心点落入的最小面积容器，面积并列为未知；同容器时gap/max(widths)≤adjacent_gap_ratio且竖直交长/min(heights)≥0.5为adjacent；无/歧义容器unknown，不从框重叠猜遮挡 | 这是首期要求模型包装器遵守的接口，不声称任意下载的ONNX都天然符合。实际模型不匹配必须增加显式适配转换并评测，不能猜输出张量。quality artifact保存算法/依赖配置，OCR artifact可为包含模型和字典的受控只读包；解包拒绝绝对路径/..和符号链接。具体模型权重及兼容runtime锁文件尚须实物验收。 -IRR-06的确定性细节、OCR行到字段及黄金输入见[事实提取规范](03-fact-extraction-contract.md)。该规范补齐当前算法，不更换D-FINE-N/PP-OCRv4，不引入LLM。参考代码是设计工具，不是已实现的服务。 +当前 [CPU 本地检测续批](../08-delivery/32-i-ml01-cpu-detection.md) 已执行真实官方权重。COCO 无 label/shelf/cabinet，因此COCO入口不产生标签OCR、化学实体或容器关系，parent_detection_id=null；OCR续批可独立识别原图文字，尚未将文字绑定到瓶子/化学字段。下述标签/关系算法是待接线要求，不能通过将任意 COCO 类改名为旧四类来填补证据。 + +IRR-06的确定性细节、OCR行到字段及黄金输入见[事实提取规范](03-fact-extraction-contract.md)。用户在2026-10-07选择PP-OCRv6_small ONNX，替代原PP-OCRv4 CPU路线；D-FINE-N和确定性事实语义保持一致，不引入LLM。实际CPU文字适配见[OCR续批](../08-delivery/33-i-ml01-ocrv6-cpu.md)，行到业务字段/实体关联仍待接线。参考代码是设计工具，不是已实现的服务。 质量未通过比较严格使用blur_scoreglare_max；恰等阈值为通过。首期自动quality只输出blur/dark/glare原因；协议occluded/unreadable保留扩展，本期不以没有定义的遮挡模型自动判失败；识别不足通过needs_review保留不确定性。 diff --git a/docs/04-ai-rules/02-model-data-plan.md b/docs/04-ai-rules/02-model-data-plan.md index eac4529..333f464 100644 --- a/docs/04-ai-rules/02-model-data-plan.md +++ b/docs/04-ai-rules/02-model-data-plan.md @@ -2,20 +2,22 @@ 设计决定：2026-09-27，ML-BASE-02。用户已选择D-FINE-N，生产允许GPU；本次固定CUDA为建议生产profile，保留CPU功能profile。本文是编码规范，不是已训练模型、环境兼容证明或上线批准。业务协议仍为vision-v1；未部署的旧模型契约由本次修订替代，不支持把旧清单改名后继续使用。 +2026-10-07 实施修订：当前只使用 CPU，CUDA 暂不安装。已交付官方 COCO 80 类本地检测入口与有界 spawn 子进程，见 [CPU 续批](../08-delivery/32-i-ml01-cpu-detection.md)。用户本次明确选择 PP-OCRv6_small ONNX，替代原 PP-OCRv4/Paddle 路线；已接本地文字检测/识别，见 [OCR 续批](../08-delivery/33-i-ml01-ocrv6-cpu.md)。下文 CUDA、完整 HTTP 流水线和生产发布仍为后续门禁。 + ## 1. 固定路线与变更边界 -| 能力 | 本期决定 | 不允许的替代 | -| ---- | ---------------------------------------------- | --------------------------------------- | -| 检测 | D-FINE-N，四类微调，640×640，300 queries | 把COCO权重直接当四类模型；未经确认换S或扩大输入 | -| 适配器 | dfine-n4-rgb-stretch-v1；RGB、直接resize、qmax、无NMS | 旧BGR/0–255/letterbox适配器；混用上游两个推理示例 | -| 生产建议 | ONNX Runtime CUDA、FP32、batch=1、并发1 | 默认TensorRT/FP16、跨主机GPU调度或静默CPU fallback | -| OCR | 保留PaddleOCR2.10.0、PP-OCRv4及CPU | 本次不隐含批准PP-OCRv6、OCR上GPU或大模型 | -| 开发 | fixture-v1联调；真实CPU profile做功能/导出检查 | 把mock、CPU结果当CUDA生产性能证明 | -| 业务 | AI只出事实；Worker写证据并执行确定性规则；仍需人工复核 | 用模型置信度代替安全判断 | +| 能力 | 本期决定 | 不允许的替代 | +| ---- | -------------------------------------------------------------------- | ---------------------------------------- | +| 检测 | 官方 D-FINE-N COCO 80 类，640×640，300 queries | 擅自压缩为四类、未经确认换 S 或扩大输入 | +| 适配器 | dfine-coco80-rgb-stretch-v1；RGB、直接resize、logits/pred_boxes、qmax/sigmoid、无NMS | 旧BGR/0–255/未核验的 letterbox 适配器；混用未匹配的推理示例 | +| 生产建议 | ONNX Runtime CUDA、FP32、batch=1、并发1 | 默认TensorRT/FP16、跨主机GPU调度或静默CPU fallback | +| OCR | 用户提供的 PP-OCRv6_small_det/rec ONNX；ORT CPU、DB/CTC、内嵌字符表 | 未核验的中型/大型权重、OCR上GPU或运行时下载 | +| 开发 | fixture-v1联调；真实CPU profile做功能/导出检查 | 把mock、CPU结果当CUDA生产性能证明 | +| 业务 | AI只出事实；Worker写证据并执行确定性规则；仍需人工复核 | 用模型置信度代替安全判断 | 上游仓库为Peterande/D-FINE，固定提交956d1709314c2c6a4df6f34de232054578a7449f；模型结构从该提交configs/dfine/dfine_hgnetv2_n_coco.yml及其include展开。source_ref固定为https://github.com/Peterande/D-FINE，source_commit为上述40位提交；不在构建时追踪master。项目包装器和安全加载改动由项目git_commit及补丁摘要追踪，不伪装成上游原样实现。 -初始化只选官方dfine_n_coco.pth，获取地址见第10节。只使用COCO版，不擅自替换obj365/obj2coco。代码LICENSE、具体权重/数据条件和现场图授权分别核查；Apache-2.0不等于所有制品无条件使用。真实权重SHA在下载核验阶段记录，不预填。 +本批直接使用用户提供并已核验的官方 COCO `model.onnx`，不把它改名成项目四类权重；如后续训练或重新导出，仍只使用 COCO 版初始化，不擅自替换 obj365/obj2coco。代码 LICENSE、具体权重/数据条件和现场图授权分别核查；Apache-2.0 不等于所有制品无条件使用。模型、config 和预处理 SHA 固定记录在 [I-ML-01 验收](../08-delivery/31-i-ml01-official-onnx-80class.md)。 ## 2. 运行时和版本约束 @@ -23,17 +25,18 @@ | 环境 | 顶层约束 | |---|---| -| AI CPU | onnxruntime1.20.1、paddleocr2.10.0、paddlepaddle2.6.2、numpy1.26.4 | -| AI CUDA | 以onnxruntime-gpu1.20.1替换CPU包；CUDA12.6系列、cuDNN9系列；OCR仍为CPU Paddle | +| 当前检测 CPU | onnxruntime1.20.1、numpy2.2.2、Pillow11.3.0；仅 CPU EP，intra=2/inter=1；Windows Python3.11.4 已执行真实模型合成图 smoke | +| 当前 OCR CPU | 同一 ORT1.20.1/numpy2.2.2；opencv-python-headless4.11.0.86、pyclipper1.3.0.post6、PyYAML6.0.2；不安装 Paddle | +| AI CUDA（暂缓） | 以onnxruntime-gpu1.20.1替换CPU包；CUDA12.6系列、cuDNN9系列；OCR ONNX会话仍明确只用CPU | | 训练/导出 | PyTorch2.8.0、torchvision0.23.0、官方cu126训练构建、ONNX1.17.0、numpy1.26.4、固定上游提交 | -| 图像 | Pillow11.3.0用于检测resize；OpenCV4.10.0.84用于共享perspective-rgb-v1；Worker与AI共享裁剪实现 | +| 图像 | Pillow11.3.0用于检测resize；本地OCR为OpenCV headless4.11.0.86；业务perspective-rgb-v1共享裁剪与依赖对照仍待接线，不把本地OCR裁剪当可重建业务证据 | -这些是明确的顶层候选约束，不是已验证安装锁。I-ML-01在空环境解析并冻结全部传递依赖、wheel hash、CUDA/cuDNN具体patch、驱动要求、镜像digest和SBOM；记录构建/离线import/smoke日志。发现冲突或不可接受漏洞，先提交版本修订ADR与回归，不让实现者静默替换。PaddleOCR涉及opencv-python/contrib，固定同版本并验证实际cv2，不再混装不同版本headless。GPU依赖由镜像提供；不使用ORT1.20未提供的preload_dlls接口。 +当前CPU版本已安装且pip check及真实模型smoke通过，记录在交付证据；这不是生产安装锁。生产仍须冻结全部传递依赖、wheel hash、目标镜像digest和SBOM；CUDA启用时另核驱动/cuDNN具体patch。当前只安装一个OpenCV headless发行包，禁止再混装python/contrib。GPU依赖由目标镜像提供；不使用ORT1.20未提供的preload_dlls接口。 | runtime_profile | device_profiles | detector_device | ocr_device | precision | |---|---|---|---|---| -| dfine-cpu-fp32-ocrv4cpu-v1 | [cpu] | cpu | cpu | fp32 | -| dfine-cuda-fp32-ocrv4cpu-v1 | [cuda] | cuda | cpu | fp32 | +| dfine-cpu-fp32-ocrv6smallcpu-v1 | [cpu] | cpu | cpu | fp32 | +| dfine-cuda-fp32-ocrv6smallcpu-v1（暂缓） | [cuda] | cuda | cpu | fp32 | | fixture-v1 | 测试部署声明的cpu/cuda路由 | mock | mock | mock | 真实runtime对象同时固定preprocess_id=dfine-rgb-stretch640-v1、postprocess_id=dfine-qmax-v1、onnx_opset=16和runtime_lock_sha256；fixture分别为fixture-v1、fixture-v1、0和测试锁文件真实摘要。一个真实bundle只声明一个设备profile；CPU/CUDA分别打包、评测和批准，不共用报告。CPU生产激活仍需独立通过cpu-functional policy；CUDA可用不自动取消CPU功能基线。 @@ -47,21 +50,18 @@ runtime-lock.json以规范JSON记录profile、两阶段设备、精度、预后 仅使用授权现场/历史照片；记录authorization_ref、来源实验室、scene_id、时间、脱敏、SHA256及撤回规则。无授权时仅用合成fixture开发；真实图、权重不入Git。 | category_id | class_id | 类型 | 标注边界 | -|---|---|---|---| -| 1 | 0 | bottle | 可见瓶体和瓶盖外接框；遮挡只标可见部分 | -| 2 | 1 | label | 可见标签外接框；无法可靠判定的实例单独标ignore供裁决 | -| 3 | 2 | shelf | 单个连续储物层/格内边界，不把整柜当一层 | -| 4 | 3 | cabinet | 可见柜体外边界，可包含多个shelf | +|---|---:|---|---| +| COCO 官方 category_id | 0–79 | 官方 COCO 80 类 | 沿用官方 COCO 目标边界和许可；业务化学事实仅消费显式支持的类别 | -授权标注原件保持COCO category_id=1..4、bbox像素xywh、area=w*h、iscrowd=0。负样本annotation为空；不增加背景类。争议标注隔离在sidecar、裁决后才进入训练/评测；未裁决实例所在图不作为无目标负样本，报告排除数量/理由。sidecar还保存目标关系、OCR字段/可读性、实体和日期真值及审核来源，不作为推理补全输入。 +官方 COCO 标注原件保持其 category_id、bbox 像素 xywh、area=w*h、iscrowd=0；模型 class_id 使用 config.json 的连续 0–79 顺序。负样本 annotation 为空；不增加背景类。争议标注隔离在 sidecar、裁决后才进入评测；报告排除数量/理由。业务 sidecar 还可保存实验室瓶、标签、位置关系和 OCR 真值，但不把这些业务类别写回官方检测头。 按scene_id分组、seed=20260927划分70%/15%/15% train/validation/test。同次摆放的总览/细节及近重复图同组，增广后不得跨组；冻结split-manifest，训练不读test。起步预算至少1000授权图、100scene、每类300实例，困难/无目标/非目标图至少20%；这不是充分性或达标保证，验收矩阵的独立样本下限仍须满足。先标100图形成手册，第二人复核至少20%及所有争议，第三人或业务专家裁决。 ### 3.2 LabSafeCOCODetection与评测桥接 -在项目训练包装器注册LabSafeCOCODetection，沿用上游图像/框变换，在向模型交付target前逐项把labels按{1:0,2:1,3:2,4:3}映射成int64。num_classes=4、remap_mscoco_category=false；禁止仅关闭映射后直接把原始1..4传给四分类头。验证标注categories集合、每个annotation类别、正面积及框边界，未知类别立即报错。 +在项目训练/评测包装器注册官方 COCO 数据集，沿用上游图像/框变换，在向模型交付 target 前按固定 category_id→class_id 表映射为 int64 0–79。num_classes=80、remap_mscoco_category=false；验证标注 categories、每个 annotation 类别、正面积及框边界，未知类别立即报错。 -COCO GT对象保持1..4不变；项目评测包装器对预测class_id反向映射0..3→1..4后交给COCO evaluator。给业务API的类别始终为bottle/label/shelf/cabinet，不进行反向映射。训练类别映射与COCO评测反向映射均有[纯语义参考](../../tools/design/dfine_reference.py)及合成往返测试。 +COCO GT 对象保持官方 category_id；项目评测包装器对预测 class_id 反向映射回官方 category_id 后交给 COCO evaluator。给推理 API 的类别保持官方 `type` 与 `class_id`，不做四类反向映射。映射和评测往返有[纯语义参考](../../tools/design/dfine_reference.py)及合成测试。 ## 4. 离线训练的首轮配方 @@ -69,7 +69,7 @@ COCO GT对象保持1..4不变；项目评测包装器对预测class_id反向映 | 项目 | 固定初值和行为 | |---|---| -| 网络 | num_classes=4、num_queries=300、eval_spatial_size=[640,640]、remap_mscoco_category=false；N型HGNetv2/encoder/decoder结构不变 | +| 网络 | num_classes=80、num_queries=300、eval_spatial_size=[640,640]、remap_mscoco_category=false；N型HGNetv2/encoder/decoder结构不变 | | 初始化 | HGNetv2.pretrained=false；显式加载本地已核验dfine_n_coco.pth，不在构造模型时另行联网下载backbone | | 优化器 | AdamW；非backbone LR=0.00025，backbone LR=0.000025；betas=[0.9,0.999]；weight_decay=0.000125；所有bias、归一化参数weight_decay=0 | | 参数组 | 每个requires_grad参数恰好进入一组；backbone前缀判别学习率、模块类型/参数名判别bias/norm；组间重复或漏参数即失败 | @@ -87,44 +87,44 @@ COCO GT对象保持1..4不变；项目评测包装器对预测class_id反向映 ### 安全加载和分类头初始化 -只能加载白名单来源和实际hash核验后的本地checkpoint，torch.load明确weights_only=true、map_location=cpu；优先取ema.module，否则取model。无这两种结构或包含非Tensor模型参数即失败。不得调用上游Objects365↔COCO类别搬运逻辑来初始化项目四类头。 +只能加载白名单来源和实际 hash 核验后的本地 checkpoint，torch.load 明确 weights_only=true、map_location=cpu；优先取 ema.module，否则取 model。无这两种结构或包含非 Tensor 模型参数即失败。不得调用上游 Objects365↔COCO 类别搬运逻辑改写官方 80 类头。 -按同名且shape一致复制参数；decoder.denoising_class_embed、decoder.enc_score_head、decoder.dec_score_head.*等类别相关参数在四类网络中重新初始化，不按前4行截取COCO类别。导出matched/missing/mismatched/unexpected清单；除这些明确的分类参数外存在未加载网络参数即停止，调查后以ADR增加精确允许项，不能用strict=false掩盖任意缺失。安全加载失败时走隔离转换审查，不静默设置weights_only=false。断点续训只接受同source/config/dataset/split哈希、同四类结构的项目checkpoint，恢复优化器/EMA/epoch；微调初始化不恢复旧COCO优化器。 +按同名且 shape 一致复制参数；类别相关参数必须与 80 类头一致，禁止按前四行截取或重排。导出 matched/missing/mismatched/unexpected 清单；存在未加载网络参数即停止，不能用 strict=false 掩盖。断点续训只接受同 source/config/dataset/split 哈希的 80 类 checkpoint。 -## 5. dfine-n4-rgb-stretch-v1适配器 +## 5. dfine-coco80-rgb-stretch-v1适配器 ### 5.1 图像输入 1. 输入为经过现有流程保存的RGB8分析图；W/H取解码图实际宽高，均为正整数。 2. Pillow Image.fromarray(rgb).resize((640,640), resample=Image.Resampling.BILINEAR)，直接拉伸，无padding，无EXIF二次旋转。 3. 结果按RGB顺序转换numpy float32后除float32(255)，转CHW、增加batch轴、保证连续内存。无BGR转换、无均值/方差标准化。 -4. ONNX输入images=float32[1,3,640,640]；orig_target_sizes=int64[1,2]，值为[[W,H]]，不是[[H,W]]、不是[[640,640]]。 +4. 本次模型输入 pixel_values=float32[1,3,640,640]；图中声明动态 batch/height/width，但本适配器固定 batch=1、640×640。原图宽高仅用于输出 cxcywh→xyxy 转换。另保留已后处理导出的 images/orig_target_sizes 签名识别，其尺寸为 [[H,W]]；不能把这个示例签名强套给本次模型。 训练/验证的PIL Resize与线上Pillow版本、插值必须相同。选择直接resize是为了统一固定提交的验证路线；不复制上游onnx_inf.py中的letterbox。OCR裁剪始终从原始分析图恢复，不能从拉伸后的640图生成证据。 ### 5.2 项目导出包装器 -加载四类项目checkpoint并model.eval；使用cfg.model.deploy()得到部署模型。包装器读取pred_logits[1,300,4]和pred_boxes[1,300,4]（归一化cxcywh）。不使用上游flatten(class×query) TopK后处理。 +真实 ORT 已核验本次官方 ONNX 输出 logits=float32[1,300,80]、pred_boxes=float32[1,300,4]（归一化 cxcywh）。适配器显式读取名称并核对实际张量 shape/dtype，不使用 flatten(class×query) TopK 后处理。示例代码的 labels/boxes/scores 属于另一种已后处理导出接口。 -- 对每query先在原始logits上求最大值和class_id；严格相等取较小class_id。confidence=sigmoid(max_logit)，不是objectness乘概率，也不是softmax。 +- 对每 query 求最大值和 class_id；严格相等取较小 class_id，confidence=sigmoid(max_logit)。本次在适配层执行，官方 ONNX 并未导出这三个后处理张量。 - 每个query只输出一个类别；不做TopK重排、不做NMS；保留300个query原始顺序，避免同query多个类别以及TopK并列不稳定。 -- 将cxcywh转换成xyxy，逐坐标乘[W,H,W,H]。不提前裁边/阈值过滤。 -- 包装器对每query检查原始logits/box有限及w/h>0；无效query把返回score置NaN，线上将其判为MODEL_ERROR。不能因无效值低于阈值而藏起来。 -- 输出labels=int64[1,300]、boxes=float32[1,300,4]（原图像素xyxy）、scores=float32[1,300]。 +- 对保留候选将cxcywh转换成xyxy，逐坐标乘[W,H,W,H]，随后按5.3裁边归一化。 +- 对每 query 的 logits 和 box 均检查有限性；NaN/Inf 无论分数高低都报 MODEL_ERROR。真实模型在黑图产生低分有限负高度框，所以先按阈值筛选，再检查保留框 w/h>0；高分退化框仍报 MODEL_ERROR。不得改写数值或伪造 NaN。 +- 当前适配器输出筛选后的检测列表；若后续重新导出已后处理包装器，其接口为labels=int64[1,300]、boxes=float32[1,300,4]（原图像素xyxy）、scores=float32[1,300]，必须单独核验，不能视为本次实物输出。 这是项目LabSafePostProcessor的明确语义，不声称与上游默认TopK完全相同。训练验证、PyTorch参考和ONNX导出共用该定义；语义参考见dfine_reference.py，实际向量化版本在I-ML-01实现。 -使用公开torch.onnx.export(dynamo=false)、opset_version=16、batch固定1，无dynamic_axes，不启用simplifier/FP16/量化；示例输入两项batch均为1，不沿用上游32张图/1组尺寸的示例。无checkpoint立即失败，禁止导出未加载预训练参数的随机网络。I-ML-01允许先保存COCO匹配参数+未训四类头的初始化checkpoint，必须标development，仅用于导出/接口原型，不冒充完成微调。运行onnx.checker并校验输入/输出名称、dtype、shape与opset；未知张量签名SCHEMA_MISMATCH，不靠猜下标读取。 +使用公开 torch.onnx.export(dynamo=false)、opset_version=16、batch 固定 1，无 dynamic_axes，不启用 simplifier/FP16/量化；示例输入两项 batch 均为 1。无 checkpoint 立即失败，禁止导出随机网络。官方 ONNX 可直接用于开发/接口原型，必须标 development，不能冒充完成现场评测。运行 onnx.checker 并校验输入/输出名称、dtype、shape 与 opset；未知张量签名 SCHEMA_MISMATCH，不靠猜下标读取。 ### 5.3 线上过滤、几何和数量限制 -在过滤前校验全部输出类别0..3、分数有限且0..1、框有限且x2>x1/y2>y1；数值损坏MODEL_ERROR，不返回空成功。scorex1/y2>y1，否则 MODEL_ERROR。框裁到[0,W]×[0,H]，裁后空框丢弃，再除[W,H,W,H]得到 API 归一化 xyxy。不得执行未声明的类别压缩或二次 NMS。 -每图按(y1,x1,y2,x2,class_id,confidence,query_index)升序稳定排序，生成现有UUIDv5 run/image/type/ordinal标识；query_index只是内部并列决胜字段，不新增API字段。不同query即使重叠也不默认合并；不同类型嵌套框不能互相NMS抑制。重复检测由四类训练和完整关联/规则验收约束，不能临时加入未版本化去重。 +每图按(y1,x1,y2,x2,class_id,confidence,query_index)升序稳定排序，生成现有 UUIDv5 run/image/type/ordinal 标识；query_index 只是内部并列决胜字段，不新增 API 字段。不同 query 即使重叠也不默认合并；不同类型嵌套框不能互相 NMS 抑制。重复检测由 80 类模型评测和完整关联/规则验收约束，不能临时加入未版本化去重。 按image ordinal处理1–3图；所有图经过阈值/裁边后的检测总数>100时HTTP500 MODEL_ERROR、不可自动重试，details说明缩小采集范围；不截断为前100，也不只保留label。用户创建范围更小的新巡检项，不能绕过现有失败项状态机。300是网络候选数，不是业务允许的最终结果数。 -label crop quad仍为矩形框TL/TR/BR/BL；宽高按原图框差向上取整，限制1..2048；继续使用perspective-rgb-v1，不宣称矩形检测提供真实透视四角。下游label→bottle、bottle→容器关联和人工复核规则保持不变并回归。 +COCO 不包含 label/shelf/cabinet。COCO本地检测报告不生成标签裁剪或容器归属，parent_detection_id 为 null，报告明确 OCR/化学实体/容器关系能力不可用。OCR续批已接自身文字检测与独立文字报告，但尚未关联COCO瓶子和明确的容器证据；不能将任意 COCO 类改名为旧四类来继续关联。 ### 5.4 导出一致性门禁 @@ -132,19 +132,21 @@ label crop quad仍为矩形框TL/TR/BR/BL；宽高按原图框差向上取整， 纯参考测试仅证明映射/qmax/几何/边界语义，不证明Pillow像素一致、真实权重或CUDA算子已验证。参考只接Python原生数值；真实numpy输出先验证dtype再tolist对照，不能先int强转来掩盖错误的类别张量。I-ML-01先做合成原型smoke；本节正式50图门禁在I-ML-02/03取得数据和训练后权重之后、I-ML-04执行，避免“先有训练结果才能开始导出原型”的循环。模型每次训练、后处理或运行环境变化均重新执行此门禁。 -## 6. OCR保持原选型 +## 6. PP-OCRv6_small ONNX CPU + +使用用户提供的 PP-OCRv6_small_det_onnx/PP-OCRv6_small_rec_onnx 两个目录。运行时只消费 inference.onnx 与 inference.yml，逐文件核验固定SHA；inference.json是导出源图资料，README只作资料，其摘要也记录在验收中，不执行其中代码或安装指示。README含中型模型示例，实际身份以已核验Global.model_name和ONNX签名为准。检测输入x=float32[1,3,H,W]、输出fetch_name_0=float32[1,1,H,W]；识别输入x=float32[1,3,48,W]、输出fetch_name_0=float32[1,W/8,18710]。 -固定资产为ch_PP-OCRv4_det_infer、ch_PP-OCRv4_rec_infer、ch_ppocr_mobile_v2.0_cls_infer、ppocr_keys_v1.txt。显式构造ocr_version=PP-OCRv4、lang=ch、use_gpu=false、use_angle_cls=true，并传det/rec/cls/字符表本地路径。启动核验每个文件；库即使关闭角度功能也可能检查cls，因此不省略资产。缺失返回MODEL_VERSION_UNAVAILABLE/not_ready，不允许运行时下载。 +适配器ppocrv6-small-db-ctc-cpu-v1：RGB转BGR；检测最长边只缩小到960、尺寸round到32倍数（至少32），OpenCV INTER_LINEAR，再按配置做/255与mean/std标准化。配置未给DetResizeForTest的具体限制，960是显式项目开发参数，尚未与Paddle上游完整流水线做黄金对照。DB按thresh=0.2、box_thresh=0.45、unclip_ratio=1.4、无dilation，生成稳定排序TL/TR/BR/BL文字四边形；超过3000轮廓或100文字区域失败，不截断。 -每个label原图crop由RGB转BGR后调用ocr(det=true,rec=true,cls=true)；None/空列表产生未知事实，不伪造识别值。逐项检查返回shape，按quad最小y、最小x排序，保留原文和confidence；未知shape为SCHEMA_MISMATCH。日期解析、实体匹配和证据关联仍遵守[事实与规则](01-data-and-rules.md)。ORT与Paddle CPU线程各固定2，MKL/OMP线程各1；单子进程内顺序运行检测与OCR，禁止每个crop创建OCR实例。CPU预算不足只允许经容量评测调整并发布新runtime lock。 +从原图做透视裁剪（INTER_LINEAR/BORDER_REPLICATE，边≤2048），高宽比≥1.5旋转90度；识别保持高48、最小宽320、8倍数对齐、最大3200，BGR/127.5−1，右侧零填充。宽度超限拒绝，不擅自压缩超长行。CTC字符表为YAML内18708项，index0为blank，最后添加space，共18710；先核shape/dtype/有限性/概率和，再去相邻重复及blank，confidence为保留字符概率均值。空白/低分行保留并标uncertain，不造化学事实。 -OCR升级或GPU化是独立变更：先用分阶段耗时证明需要，再选择具体模型/运行时、重做依赖和联合显存验证；不能在本次D-FINE清单中暗换模型。 +两会话只加载一次、唯一CPUExecutionProvider、intra=2/inter=1/sequential、关闭fallback。没有提供角度分类器，当前不宣称自动纠正180度/任意页面方向；不存在运行时下载。当前可识别原图文字而不依赖COCO的label类别，但未将行关联到瓶子或业务字段。日期解析、实体匹配和证据关联仍须按[事实与规则](01-data-and-rules.md)接线；文字不是已确认的化学品/到期日。模型改动和后续GPU化须产生新runtime/制品摘要并重新评测，不沿用旧v4报告。 ## 7. 制品、hash与发布门禁 ModelManifest保留detector/ocr/quality/dictionary四个角色，每角色恰一份。各角色object_key指向版本化不可变制品，其sha256对下载原始字节计算。detector使用受控tar包，包含detector.onnx、adapter.json、source.json、runtime-lock.json、files.sha256.json及LICENSE/NOTICE；不打包训练checkpoint。adapter.json存输入输出签名、类别双向映射、resize、qmax和容量规则；source.json存上游提交、项目git_commit、训练配置/数据split摘要及初始化权重实际hash。files.sha256.json列出除自身外所有文件的实际hash，外层tar哈希覆盖索引本身。 -OCR角色为模型/cls/keys及其文件索引包；quality角色为算法和图像依赖配置；dictionary角色为冻结词典快照。解包拒绝绝对路径、..、符号/硬链接、重复路径；每包最多4096文件、累计解压最多1GiB，越界MODEL_VERSION_UNAVAILABLE。校验完整文件清单后原子放入只读模型目录，不从归档执行代码。runtime_lock_sha256必须等于detector包中runtime-lock.json实际字节摘要，内容还须与runtime对象和当前环境匹配；检查字段非空不等于校验制品。 +OCR角色为det/rec ONNX与各自YAML（含字符表）的文件索引包；当前不含cls。quality角色为算法和图像依赖配置；dictionary角色为冻结词典快照。解包拒绝绝对路径、..、符号/硬链接、重复路径；每包最多4096文件、累计解压最多1GiB，越界MODEL_VERSION_UNAVAILABLE。校验完整文件清单后原子放入只读模型目录，不从归档执行代码。runtime_lock_sha256必须等于detector包中runtime-lock.json实际字节摘要，内容还须与runtime对象和当前环境匹配；检查字段非空不等于校验制品。 content_sha256对下列16项按共享规范JSON计算：artifacts、pipeline_version、dictionary_version_id、dictionary_sha256、thresholds、input_max_side、adapter_id、model_family、source_ref、source_commit、git_commit、detector_backend、ocr_backend、device_profiles、runtime_profile、runtime。评测绑定content_sha256；批准台账绑定完整manifest/report/policy哈希。runtime lock或代码/预后处理变动必须产生新评测内容，不能沿用旧CUDA报告。校准报告为独立sidecar，真实发布从受控制品登记取出、核其report/split摘要，不增加第五个artifact角色。 @@ -162,13 +164,15 @@ FP32基线禁用TF32优化（ORT CUDA provider use_tf32=0），不自动启用FP ## 9. 编码任务及完成标准 +当前[联合CPU续批](../08-delivery/34-i-ml01-quality-cpu-pipeline.md)已串联真实quality像素、COCO检测与OCR文字识别并执行合成图门禁/重放，默认CPU、不安装CUDA；尚未接受控analysis读取、完整HTTP、化学事实或生产bundle。 + 以下是未来实现交付，不表示apps或模型文件已存在。纯参考代码和合成测试属于设计工具。 | 任务 | 必须实现/交付 | 失败处理 | |---|---|---| -| I-ML-01 | 固定提交和项目补丁；训练/CPU/CUDA lock；LabSafeCOCODetection/评测反向映射、安全加载、LabSafePostProcessor、初始化开发checkpoint、固定ONNX导出和合成CPU/CUDA smoke | 不兼容形成有证据ADR；初始化权重仅development，不冒充训练后制品 | +| I-ML-01 | 官方 COCO 80 类 artifact、输入输出签名、安全加载、固定 ONNX provider 和合成 smoke | 不兼容形成有证据 ADR；未完成真实 ORT/CUDA 评测前不得声称生产达标 | | I-ML-02 | 数据授权/标注手册/scene划分/争议裁决/冻结split | 不足继续采集，无授权只用fixture联调 | -| I-ML-03 | 四类微调、最佳EMA、resolved-config、validation阈值、受控制品包 | 目标不达标继续分析/训练，不读取test挑结果 | +| I-ML-03 | 官方 80 类模型校准、resolved-config、validation 阈值、受控制品包 | 目标不达标继续分析/训练，不读取 test 挑结果 | | I-ML-04 | 训练后50图PyTorch/ORT CPU/CUDA对照、既有RPC、多图与100上限、原图裁剪/OCR/关系/规则；GPU缺失/OOM/超时/重载/断网验证 | 失败不写部分事实、不静默降级；报告分阶段延迟和资源 | | R-ML-01 | 冻结test实际报告、许可/SBOM扫描、目标设备环境快照、独立批准台账 | 缺证据禁止生产，不改变proposed状态冒充批准 | @@ -189,9 +193,10 @@ FP32基线禁用TF32优化（ORT CUDA provider use_tf32=0），不自动启用FP checkpoint实现：src/solver/_solver.py ORT CUDA兼容性：https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html ORT执行与fallback：https://onnxruntime.ai/docs/api/python/api_summary.html -PaddleOCR2.10：https://www.paddleocr.ai/v2.10.0/en/ppocr/quick_start.html +PP-OCRv6_small_det ONNX：https://huggingface.co/PaddlePaddle/PP-OCRv6_small_det_onnx +PP-OCRv6_small_rec ONNX：https://huggingface.co/PaddlePaddle/PP-OCRv6_small_rec_onnx Pillow11.3.0：https://pypi.org/project/pillow/11.3.0/ PyTorch版本配对：https://pytorch.org/get-started/previous-versions/ ~~~ -本轮没有下载实际权重、训练、安装GPU环境或生成真实性能成绩。源码存在、合成测试通过、设计可编码与环境可运行/模型达标/获准上线分别判断。 +本批核验用户提供的D-FINE与PP-OCRv6_small ONNX及运行配置SHA，在ORT CPU完成合成图检测、文字识别、重放和子进程检查。没有安装CUDA；真实照片准确率、文字到化学事实关联、完整业务RPC和生产性能尚未验收。源码、合成图执行成功、模型达标和获准上线分别判断。 diff --git a/docs/04-ai-rules/03-fact-extraction-contract.md b/docs/04-ai-rules/03-fact-extraction-contract.md index d46d75d..1dfa629 100644 --- a/docs/04-ai-rules/03-fact-extraction-contract.md +++ b/docs/04-ai-rules/03-fact-extraction-contract.md @@ -2,12 +2,14 @@ 决定：IRR-06，2026-09-27。补齐ML-BASE-02现有流程，不改变模型或业务能力。所有算法共享版本和黄金样例；参考为[readiness_reference.py](../../tools/design/readiness_reference.py)，测试为[test_readiness_design.py](../../tools/design/test_readiness_design.py)。这两个文件不是应用实现，纯数值测试不证明真实Pillow/OpenCV/Paddle输出已验证。 +2026-10-07实施更新：用户已选择PP-OCRv6_small ONNX CPU；[OCR续批](../08-delivery/33-i-ml01-ocrv6-cpu.md)完成本地文字检测/识别，[联合CPU续批](../08-delivery/34-i-ml01-quality-cpu-pipeline.md)完成下述真实质量像素算法与检测/OCR门禁。字段词法、文字到瓶子/证据闭包和聚合事实仍待接线，不能把本地行报告直接视为OCRField或化学事实。 + ## 1. quality-rgb-lap1-v1 1. 输入已按既定流程解码、应用EXIF且去元数据的RGB8分析图；不再旋转或转BGR。 2. M=max(W,H)。M≤1024不resize；否则w=max(1,(W×1024+floor(M/2))//M)，h同理。Pillow11.3.0、Image.Resampling.BILINEAR、无padding，不放大小图。 3. 灰度逐像素计算Y=(77R+150G+29B+128)//256。先转uint32防溢出，结果uint8；不使用另一套库默认灰度系数。 -4. Laplacian核为[[0,1,0],[1,-4,1],[0,1,0]]。输入转float64，边界REFLECT_101；宽/高为1时该轴重复唯一像素。等价OpenCV4.10.0.84的Laplacian(ddepth=CV_64F,ksize=1,scale=1,delta=0,borderType=BORDER_REFLECT_101)，不得用CV_8U截负值。 +4. Laplacian核为[[0,1,0],[1,-4,1],[0,1,0]]。输入转float64，边界REFLECT_101；宽/高为1时该轴重复唯一像素。等价OpenCV的Laplacian(ddepth=CV_64F,ksize=1,scale=1,delta=0,borderType=BORDER_REFLECT_101)，不得用CV_8U截负值。本批NumPy实现已与实际OpenCV4.11独立核对照，包括边界和单像素轴；quality执行本身不依赖OpenCV。 5. blur_score=所有Laplacian像素的总体方差（ddof=0）；brightness=mean(Y)/255；glare_ratio=count(Y≥250)/(w×h)。float64累计，禁止先ROUND。 6. blur_scoreglare_max分别加blur/dark/glare；恰等通过。多原因按上述顺序，任一原因即needs_retake。 @@ -23,7 +25,7 @@ overview全部bottle按detection_id排序生成i 10000 + or rgb.shape[0] * rgb.shape[1] > 40_000_000 + ): + raise CropError("SCHEMA_MISMATCH", "Expected bounded RGB8 pixels") + + +def perspective_rgb(rgb, recipe): + """Use the serialized normalized recipe as the sole source of crop geometry.""" + import cv2 + import numpy as np + + _rgb(rgb) + if not isinstance(recipe, CropTransform): + raise CropError("SCHEMA_MISMATCH", "Expected a crop transform") + recipe.validate() + height, width = rgb.shape[:2] + # Multiply in float64, then cast once. Both processes follow this path. + points = (np.asarray(recipe.quad, dtype=np.float64) * [width - 1, height - 1]).astype( + np.float32 + ) + edges = np.roll(points, -1, axis=0) - points + following = np.roll(edges, -1, axis=0) + cross = edges[:, 0] * following[:, 1] - edges[:, 1] * following[:, 0] + if not (cross > 0).all() or cv2.contourArea(points) < 1: + raise CropError("MODEL_ERROR", "Crop must be convex, clockwise and have pixel area") + out_w, out_h = recipe.output_width, recipe.output_height + target = np.array([[0, 0], [out_w - 1, 0], [out_w - 1, out_h - 1], [0, out_h - 1]], np.float32) + matrix = cv2.getPerspectiveTransform(points, target) + if not np.isfinite(matrix).all() or abs(np.linalg.det(matrix)) < 1e-8: + raise CropError("MODEL_ERROR", "Crop transform is singular") + try: + crop = cv2.warpPerspective( + rgb, + matrix, + (out_w, out_h), + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0), + ) + except cv2.error as error: + raise CropError("MODEL_ERROR", "Crop transform failed") from error + return np.ascontiguousarray(crop) + + +def encode_rgb_png(rgb): + """Fresh RGB image strips metadata; fixed PNG encoding for byte-level checks.""" + from PIL import Image + + _rgb(rgb) + with io.BytesIO() as stream, Image.fromarray(rgb) as image: + image.save(stream, format="PNG", compress_level=9, optimize=False) + return stream.getvalue() + + +@dataclass(frozen=True) +class CropEvidence: + pixels: object + png: bytes + sha256: str + + +def rebuild_crop(rgb, recipe, *, expected_sha256=None): + """Return pixels/bytes; fail closed if the caller's evidence digest differs.""" + pixels = perspective_rgb(rgb, recipe) + png = encode_rgb_png(pixels) + digest = hashlib.sha256(png).hexdigest() + if expected_sha256 is not None and expected_sha256 != digest: + raise CropError("HASH_MISMATCH", "Rebuilt crop hash differs") + return CropEvidence(pixels, png, digest) + + +def recognition_rgb(pixels, rotation): + """Explicit OCR-only CCW rotation; the evidence PNG keeps source orientation.""" + import numpy as np + + _rgb(pixels) + if type(rotation) is not int or rotation not in (0, 90): + raise CropError("SCHEMA_MISMATCH", "Unsupported recognition rotation") + return np.ascontiguousarray(np.rot90(pixels) if rotation == 90 else pixels) + + +def rgb_digest(pixels): + import numpy as np + + _rgb(pixels) + height, width = pixels.shape[:2] + header = f"rgb8:{width}:{height}:".encode("ascii") + digest = hashlib.sha256(header) + digest.update(memoryview(np.ascontiguousarray(pixels))) + return digest.hexdigest() + + +def rebuild_ocr_evidence(rgb, metadata): + """Verify source, PNG and the exact oriented RGB input used for recognition.""" + required = { + "recipe", + "png_sha256", + "png_size_bytes", + "source_rgb_sha256", + "recognition_rotation_ccw", + "recognition_width", + "recognition_height", + "recognition_rgb_sha256", + } + if not isinstance(metadata, dict) or not required <= metadata.keys(): + raise CropError("SCHEMA_MISMATCH", "Incomplete OCR crop evidence") + for key in ("png_sha256", "source_rgb_sha256", "recognition_rgb_sha256"): + digest = metadata[key] + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(c not in "0123456789abcdef" for c in digest) + ): + raise CropError("SCHEMA_MISMATCH", "Invalid evidence digest") + for key in ("png_size_bytes", "recognition_width", "recognition_height"): + if type(metadata[key]) is not int or metadata[key] < 1: + raise CropError("SCHEMA_MISMATCH", "Invalid evidence dimensions or size") + if rgb_digest(rgb) != metadata["source_rgb_sha256"]: + raise CropError("HASH_MISMATCH", "Source RGB hash differs") + recipe = CropTransform.from_dict(metadata["recipe"]) + evidence = rebuild_crop(rgb, recipe, expected_sha256=metadata["png_sha256"]) + rotation = metadata["recognition_rotation_ccw"] + oriented = recognition_rgb(evidence.pixels, rotation) + expected_rotation = 90 if recipe.output_height / recipe.output_width >= 1.5 else 0 + if rotation != expected_rotation: + raise CropError("SCHEMA_MISMATCH", "Recognition rotation differs from the crop policy") + if ( + len(evidence.png) != metadata["png_size_bytes"] + or oriented.shape[:2] != (metadata["recognition_height"], metadata["recognition_width"]) + or rgb_digest(oriented) != metadata["recognition_rgb_sha256"] + ): + raise CropError("HASH_MISMATCH", "Recognition pixels or PNG size differ") + return evidence + + +def crop_runtime(): + import cv2 + from PIL import features + + return { + "transform_version": TRANSFORM_VERSION, + "png_encoding": PNG_ENCODING, + "packages": {name: version(name) for name in ("numpy", "Pillow", "opencv-python-headless")}, + "opencv_build_sha256": hashlib.sha256(cv2.getBuildInformation().encode()).hexdigest(), + "zlib_version": features.version("zlib"), + "point_scaling": "float64-widthminus1-heightminus1-then-float32", + "interpolation": "INTER_LINEAR", + "border": "BORDER_CONSTANT-rgb0", + "recognition_rotation": "ccw90-if-height-over-width-ge1.5", + "recognition_digest": "sha256-ascii-rgb8:width:height:-then-c-order-rgb-bytes", + } diff --git a/packages/inference_protocol/contract.json b/packages/inference_protocol/contract.json index 0e49fa9..67f3f59 100644 --- a/packages/inference_protocol/contract.json +++ b/packages/inference_protocol/contract.json @@ -1028,13 +1028,94 @@ } ] }, + "class_id": { + "type": "integer", + "minimum": 0, + "maximum": 79 + }, "type": { "type": "string", "enum": [ + "person", + "bicycle", + "car", + "motorbike", + "aeroplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", "bottle", - "label", - "shelf", - "cabinet" + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "sofa", + "pottedplant", + "bed", + "diningtable", + "toilet", + "tvmonitor", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush" ] }, "bbox": { @@ -1057,6 +1138,7 @@ "detection_id", "image_id", "parent_detection_id", + "class_id", "type", "bbox", "confidence" @@ -1563,15 +1645,15 @@ "adapter_id": { "type": "string", "enum": [ - "dfine-n4-rgb-stretch-v1", + "dfine-coco80-rgb-stretch-v1", "fixture-v1" ] }, "runtime_profile": { "type": "string", "enum": [ - "dfine-cpu-fp32-ocrv4cpu-v1", - "dfine-cuda-fp32-ocrv4cpu-v1", + "dfine-cpu-fp32-ocrv6smallcpu-v1", + "dfine-cuda-fp32-ocrv6smallcpu-v1", "fixture-v1" ] }, diff --git a/pyproject.toml b/pyproject.toml index 055a395..c802c78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,18 @@ ai = [ "jsonschema==4.23.0", "rfc3339-validator==0.1.4", ] +ai-real = [ + "numpy==2.2.2", + "Pillow==11.3.0", + "onnxruntime==1.20.1", + "opencv-python-headless==4.11.0.86", + "pyclipper==1.3.0.post6", +] +image-evidence = [ + "numpy==2.2.2", + "Pillow==11.3.0", + "opencv-python-headless==4.11.0.86", +] api = [ "fastapi==0.115.6", "pydantic==2.10.5", diff --git a/requirements/py311-ai-real.txt b/requirements/py311-ai-real.txt new file mode 100644 index 0000000..c20d041 --- /dev/null +++ b/requirements/py311-ai-real.txt @@ -0,0 +1,5 @@ +# Optional real-model runtime; keep this separate from the fixture CI environment. +-r py311-ai.txt +-r py311-image-evidence.txt +onnxruntime==1.20.1 +pyclipper==1.3.0.post6 diff --git a/requirements/py311-image-evidence.txt b/requirements/py311-image-evidence.txt new file mode 100644 index 0000000..db157aa --- /dev/null +++ b/requirements/py311-image-evidence.txt @@ -0,0 +1,4 @@ +# Shared crop reconstruction runtime; no ONNX, CUDA or business dependencies. +numpy==2.2.2 +Pillow==11.3.0 +opencv-python-headless==4.11.0.86 diff --git a/tests/ai/test_cpu_pixels.py b/tests/ai/test_cpu_pixels.py new file mode 100644 index 0000000..8ae34da --- /dev/null +++ b/tests/ai/test_cpu_pixels.py @@ -0,0 +1,48 @@ +import unittest + +from apps.ai_inference.adapters.dfine import AdapterError, postprocess_project, preprocess_rgb + +try: + import numpy as np + from PIL import Image +except ImportError: + np = Image = None + + +@unittest.skipIf(np is None, "Pixel tests require the independent real CPU runtime") +class CpuPixelTests(unittest.TestCase): + def test_non_square_rgb_chw_float32_and_height_width_target(self): + source = np.zeros((400, 800, 3), dtype=np.uint8) + source[:, :400, 0] = 255 + source[:, 400:, 2] = 255 + with Image.fromarray(source) as image: + tensor, sizes = preprocess_rgb(image) + self.assertEqual(tensor.shape, (1, 3, 640, 640)) + self.assertEqual(str(tensor.dtype), "float32") + self.assertTrue(tensor.flags.c_contiguous) + self.assertEqual(sizes.tolist(), [[400, 800]]) + self.assertEqual(str(sizes.dtype), "int64") + self.assertEqual(tensor[0, :, 0, 0].tolist(), [1.0, 0.0, 0.0]) + self.assertEqual(tensor[0, :, 0, -1].tolist(), [0.0, 0.0, 1.0]) + + def test_grayscale_is_not_silently_accepted(self): + with Image.new("L", (80, 40)) as image: + with self.assertRaises(AdapterError): + preprocess_rgb(image) + + def test_raw_tensor_nonfinite_and_no_silent_truncation(self): + logits = np.full((1, 300, 80), -20, dtype=np.float32) + boxes = np.full((1, 300, 4), 0.5, dtype=np.float32) + logits[0, :100, 39] = 8 + self.assertEqual(len(postprocess_project(logits, boxes, [800, 400], 0.4)), 100) + logits[0, 100, 39] = 8 + with self.assertRaises(AdapterError): + postprocess_project(logits, boxes, [800, 400], 0.4) + logits[0, 100, 39] = -20 + boxes[0, 299, 0] = np.nan + with self.assertRaises(AdapterError): + postprocess_project(logits, boxes, [800, 400], 0.4) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ai/test_cpu_runner.py b/tests/ai/test_cpu_runner.py new file mode 100644 index 0000000..07bb41b --- /dev/null +++ b/tests/ai/test_cpu_runner.py @@ -0,0 +1,190 @@ +import json +import multiprocessing +import os +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +from uuid import uuid4 + +from apps.ai_inference.adapters.dfine import AdapterError +from apps.ai_inference.cpu import CpuOptions, _compute, _decode_local, run_cpu_detection + +try: + from PIL import Image +except ImportError: + Image = None + + +def delayed_child(channel, *args): + time.sleep(10) + channel.send(("result", {"late": True})) + channel.close() + + +def successful_child(channel, *args): + channel.send(("result", {"complete": True})) + channel.close() + + +def crashed_child(channel, *args): + channel.close() + + +class CpuProcessTests(unittest.TestCase): + def test_timeout_kills_child_and_next_run_can_succeed(self): + before = {child.pid for child in multiprocessing.active_children()} + options = CpuOptions("model", "config", "preprocessor", timeout_seconds=1) + started = time.monotonic() + with ( + patch.dict(os.environ, APP_ENV="test"), + patch("apps.ai_inference.cpu._child", delayed_child), + ): + with self.assertRaises(AdapterError) as error: + run_cpu_detection(options, ["unused.png"], str(uuid4())) + self.assertEqual(error.exception.code, "AI_TIMEOUT") + self.assertLess(time.monotonic() - started, 5) + self.assertEqual({child.pid for child in multiprocessing.active_children()}, before) + with ( + patch.dict(os.environ, APP_ENV="test"), + patch("apps.ai_inference.cpu._child", successful_child), + ): + self.assertEqual( + run_cpu_detection(CpuOptions("m", "c", "p"), ["unused.png"], str(uuid4())), + {"complete": True}, + ) + + def test_child_crash_has_no_success_or_orphan(self): + before = {child.pid for child in multiprocessing.active_children()} + with ( + patch.dict(os.environ, APP_ENV="test"), + patch("apps.ai_inference.cpu._child", crashed_child), + ): + with self.assertRaises(AdapterError) as error: + run_cpu_detection(CpuOptions("m", "c", "p"), ["unused.png"], str(uuid4())) + self.assertEqual(error.exception.code, "MODEL_ERROR") + self.assertEqual({child.pid for child in multiprocessing.active_children()}, before) + + def test_config_and_environment_fail_before_spawn(self): + for options in ( + CpuOptions("m", "c", "p", detection_min=float("nan")), + CpuOptions("m", "c", "p", detection_min=True), + CpuOptions("m", "c", "p", threads=0), + CpuOptions("m", "c", "p", timeout_seconds=float("inf")), + CpuOptions("m", "c", "p", timeout_seconds=181), + ): + with self.subTest(options=options), patch.dict(os.environ, APP_ENV="test"): + with self.assertRaises(AdapterError): + run_cpu_detection(options, ["unused.png"], str(uuid4())) + with patch.dict(os.environ, APP_ENV="production"): + with self.assertRaises(AdapterError): + run_cpu_detection(CpuOptions("m", "c", "p"), ["unused.png"], str(uuid4())) + with patch.dict(os.environ, APP_ENV="test"): + for paths, run_id in (([], str(uuid4())), (["x"] * 4, str(uuid4())), (["x"], "bad")): + with self.assertRaises(AdapterError): + run_cpu_detection(CpuOptions("m", "c", "p"), paths, run_id) + + def test_cli_failure_does_not_write_report_or_expose_input_path(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "report.json" + secret_path = Path(directory) / "private-image.png" + result = subprocess.run( + [ + sys.executable, + "-m", + "apps.ai_inference.detect", + "--model", + "missing.onnx", + "--input", + str(secret_path), + "--output", + str(output), + ], + env={**os.environ, "APP_ENV": "production"}, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(json.loads(result.stderr)["error"]["code"], "VALIDATION_ERROR") + self.assertNotIn(str(secret_path), result.stderr) + self.assertFalse(output.exists()) + + +@unittest.skipIf(Image is None, "Real CPU image tests require requirements/py311-ai-real.txt") +class CpuImageTests(unittest.TestCase): + def test_decode_normalizes_exif_once_and_rejects_animation_and_truncation(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "original.jpg" + exif = Image.Exif() + exif[274] = 6 + with Image.new("RGB", (80, 40)) as original: + original.save(path, exif=exif) + image, digest = _decode_local(path) + self.assertEqual(image.size, (40, 80)) + self.assertEqual(len(digest), 64) + self.assertEqual(image.mode, "RGB") + image.close() + path.write_bytes(b"not-an-image") + with self.assertRaises(AdapterError): + _decode_local(path) + with Image.new("RGB", (8, 8)) as first, Image.new("RGB", (8, 8), "red") as second: + first.save(root / "animation.png", save_all=True, append_images=[second]) + with self.assertRaises(AdapterError): + _decode_local(root / "animation.png") + + def test_model_loaded_once_ids_stable_unknown_business_and_batch_capacity(self): + class Detector: + providers = ("CPUExecutionProvider",) + evidence = SimpleNamespace( + model_sha256="a" * 64, config_sha256="b" * 64, preprocessor_sha256="c" * 64 + ) + count = 1 + + def detect(self, image, threshold, *, diagnostics=None): + if diagnostics is not None: + diagnostics.update(candidates=300, kept=self.count) + return [ + {"class_id": 0, "type": "person", "bbox": [0, 0, 1, 1], "confidence": 0.8} + for _ in range(self.count) + ] + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "image.png" + with Image.new("RGB", (80, 40)) as image: + image.save(path) + options = CpuOptions("m", "c", "p") + run_id = str(uuid4()) + detector = Detector() + with ( + patch( + "apps.ai_inference.cpu.OnnxDetector.from_path", return_value=detector + ) as load, + patch("apps.ai_inference.cpu.version", return_value="test"), + ): + result = _compute(options, [path, path], run_id) + load.assert_called_once() + again = _compute(options, [path, path], run_id) + self.assertEqual( + result["images"][0]["detections"], again["images"][0]["detections"] + ) + self.assertNotEqual( + result["images"][0]["image_id"], result["images"][1]["image_id"] + ) + self.assertTrue(result["review_required"]) + self.assertFalse(result["is_simulated"]) + self.assertFalse(result["business_capabilities"]["chemical_entities"]) + self.assertIsNone(result["images"][0]["detections"][0]["parent_detection_id"]) + detector.count = 60 + with self.assertRaises(AdapterError) as error: + _compute(options, [path, path], run_id) + self.assertEqual(error.exception.code, "MODEL_ERROR") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ai/test_crop_evidence.py b/tests/ai/test_crop_evidence.py new file mode 100644 index 0000000..7e52f85 --- /dev/null +++ b/tests/ai/test_crop_evidence.py @@ -0,0 +1,230 @@ +"""Reconstruction, independent pixel goldens and fail-closed crop evidence.""" + +import copy +import importlib.util +import io +import json +import subprocess +import sys +import unittest + +from packages.image_evidence.perspective import CropError, CropTransform + +REAL = all(importlib.util.find_spec(name) for name in ("numpy", "PIL", "cv2")) + + +class CropBoundaryTests(unittest.TestCase): + def test_import_does_not_load_models_business_or_numerical_packages(self): + result = subprocess.run( + [ + sys.executable, + "-c", + "import sys; import packages.image_evidence.perspective; " + "assert not set(sys.modules) & {'numpy','cv2','PIL','onnxruntime','boto3'," + "'sqlalchemy','apps.ai_inference','packages.domain'}", + ], + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_schema_rejects_bool_nonfinite_size_version_and_bad_points(self): + valid = CropTransform(((0, 0), (1, 0), (1, 1), (0, 1)), 20, 10).as_dict() + cases = [] + for key, values in ( + ("output_width", (0, 2049, True, 2.0)), + ("output_height", (-1, 2049)), + ("transform_version", ("perspective-v2", None)), + ("quad", ([], [[0, 0]] * 4, [{"x": 0, "y": 0, "z": 0}] * 4)), + ): + cases.extend({**valid, key: value} for value in values) + for number in (True, float("nan"), float("inf"), -0.01, 1.01, "0"): + bad = copy.deepcopy(valid) + bad["quad"][0]["x"] = number + cases.append(bad) + cases += [{**valid, "rotation": 90}, {k: v for k, v in valid.items() if k != "quad"}] + for value in cases: + with self.subTest(value=value), self.assertRaises(CropError) as caught: + CropTransform.from_dict(value) + self.assertEqual(caught.exception.code, "SCHEMA_MISMATCH") + + +@unittest.skipUnless(REAL, "Crop pixels require the real CPU environment") +class CropPixelTests(unittest.TestCase): + def test_identity_rgb_png_metadata_and_byte_replay(self): + import numpy as np + from PIL import Image + + from packages.image_evidence.perspective import rebuild_crop + + rgb = np.random.default_rng(35).integers(0, 256, (19, 37, 3), np.uint8) + recipe = CropTransform(((0, 0), (1, 0), (1, 1), (0, 1)), 37, 19) + first = rebuild_crop(rgb, recipe) + restored = CropTransform.from_dict(json.loads(json.dumps(recipe.as_dict()))) + second = rebuild_crop(rgb, restored, expected_sha256=first.sha256) + np.testing.assert_array_equal(first.pixels, rgb) + self.assertEqual(first.png, second.png) + with Image.open(io.BytesIO(first.png)) as image: + self.assertEqual(image.mode, "RGB") + self.assertEqual(image.info, {}) + np.testing.assert_array_equal(np.asarray(image), rgb) + + def test_subrectangle_pixel_golden_uses_width_minus_one_on_non_square_source(self): + import numpy as np + + from packages.image_evidence.perspective import perspective_rgb + + y, x = np.indices((9, 13)) + rgb = np.stack((x * 10, y * 20, x + y), axis=-1).astype(np.uint8) + recipe = CropTransform( + ((2 / 12, 1 / 8), (8 / 12, 1 / 8), (8 / 12, 5 / 8), (2 / 12, 5 / 8)), 7, 5 + ) + np.testing.assert_array_equal(perspective_rgb(rgb, recipe), rgb[1:6, 2:9]) + + def test_skew_crop_matches_independent_opencv_geometry(self): + import cv2 + import numpy as np + + from packages.image_evidence.perspective import perspective_rgb + + rgb = np.random.default_rng(36).integers(0, 256, (70, 130, 3), np.uint8) + source = np.array([[5, 2], [118, 14], [99, 60], [14, 49]], np.float32) + target = np.array([[0, 0], [99, 0], [99, 39], [0, 39]], np.float32) + expected = cv2.warpPerspective( + rgb, + cv2.getPerspectiveTransform(source, target), + (100, 40), + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0), + ) + recipe = CropTransform( + tuple(map(tuple, (source.astype(np.float64) / [129, 69]).tolist())), 100, 40 + ) + np.testing.assert_array_equal(perspective_rgb(rgb, recipe), expected) + + def test_invalid_geometry_singular_one_pixel_and_rgb_rejected(self): + import numpy as np + + from packages.image_evidence.perspective import perspective_rgb + + rgb = np.zeros((40, 60, 3), np.uint8) + for quad in ( + ((0, 0), (1, 1), (1, 0), (0, 1)), + ((0, 0), (0, 1), (1, 1), (1, 0)), + ((0, 0), (1, 0), (1, 0), (0, 1)), + ((0, 0), (0.001, 0), (0.001, 0.001), (0, 0.001)), + ): + with self.subTest(quad=quad), self.assertRaises(CropError) as caught: + perspective_rgb(rgb, CropTransform(quad, 10, 10)) + self.assertEqual(caught.exception.code, "MODEL_ERROR") + for size in ((1, 10), (10, 1)): + with self.assertRaises(CropError): + perspective_rgb(rgb, CropTransform(((0, 0), (1, 0), (1, 1), (0, 1)), *size)) + for bad in (rgb.astype(np.float32), rgb[:, :, 0], np.zeros((1, 10, 3), np.uint8)): + with self.assertRaises(CropError): + perspective_rgb(bad, CropTransform(((0, 0), (1, 0), (1, 1), (0, 1)), 10, 10)) + + def _ocr_crop(self, tall=False): + import numpy as np + + from apps.ai_inference.adapters.ocrv6 import crop_with_evidence + from packages.image_evidence.perspective import rgb_digest + + rgb = np.random.default_rng(35).integers(0, 256, (90, 140, 3), np.uint8) + points = np.array( + [[10, 5], [30, 5], [30, 75], [10, 75]] + if tall + else [[10, 5], [125, 5], [125, 40], [10, 40]], + np.float32, + ) + recognized, metadata = crop_with_evidence(rgb, points) + metadata["source_rgb_sha256"] = rgb_digest(rgb) + return rgb, recognized, metadata + + def test_serialized_tall_evidence_rebuild_preserves_recognition_pixels(self): + import numpy as np + + from packages.image_evidence.perspective import rebuild_ocr_evidence + + rgb, recognized, metadata = self._ocr_crop(tall=True) + restored = json.loads(json.dumps(metadata)) + evidence = rebuild_ocr_evidence(rgb, restored) + self.assertEqual(metadata["recognition_rotation_ccw"], 90) + self.assertEqual(evidence.pixels.shape, (70, 20, 3)) + np.testing.assert_array_equal(recognized, np.rot90(evidence.pixels)) + self.assertTrue(recognized.flags.c_contiguous) + + def test_ocr_recognition_tensor_is_derived_from_the_reported_evidence(self): + from unittest.mock import patch + + import numpy as np + from PIL import Image + + from apps.ai_inference.adapters.ocrv6 import OnnxOCR, rec_tensor + from packages.image_evidence.perspective import rebuild_ocr_evidence, recognition_rgb + + rgb = np.random.default_rng(37).integers(0, 256, (90, 140, 3), np.uint8) + points = np.array([[10, 5], [30, 5], [30, 75], [10, 75]], np.float32) + ocr = OnnxOCR.__new__(OnnxOCR) + ocr.det, ocr.rec = object(), object() + ocr.characters = ("", "a", *("" for _ in range(18708))) + fed = [] + + def run(session, tensor): + if session is ocr.det: + return np.zeros((1, 1, *tensor.shape[2:]), np.float32) + fed.append(tensor.copy()) + output = np.zeros((1, tensor.shape[3] // 8, 18710), np.float32) + output[:, :, 0] = 1 + output[0, 1, 0], output[0, 1, 1] = 0, 1 + return output + + with ( + Image.fromarray(rgb) as image, + patch.object(ocr, "_run", side_effect=run), + patch( + "apps.ai_inference.adapters.ocrv6.db_regions", + return_value=[{"quad_pixels": points, "detection_confidence": 0.8}], + ), + ): + rows = ocr.recognize(image) + self.assertEqual(rows[0]["text"], "a") + metadata = json.loads(json.dumps(rows[0]["crop_evidence"])) + rebuilt = rebuild_ocr_evidence(rgb, metadata) + oriented = recognition_rgb(rebuilt.pixels, metadata["recognition_rotation_ccw"]) + np.testing.assert_array_equal(fed[0], rec_tensor(oriented)) + + def test_tampered_source_recipe_png_rotation_and_recognition_digest_fail_closed(self): + from packages.image_evidence.perspective import rebuild_ocr_evidence + + rgb, _, metadata = self._ocr_crop() + for key, value in ( + ("source_rgb_sha256", "0" * 64), + ("png_sha256", "0" * 64), + ("png_size_bytes", 1), + ("recognition_rgb_sha256", "0" * 64), + ("recognition_width", 1), + ("recognition_rotation_ccw", 180), + ("recognition_rotation_ccw", 90), + ("recognition_rotation_ccw", False), + ): + with self.subTest(key=key, value=value), self.assertRaises(CropError): + rebuild_ocr_evidence(rgb, {**metadata, key: value}) + bad = copy.deepcopy(metadata) + bad["recipe"]["quad"][0]["x"] += 0.01 + with self.assertRaises(CropError) as caught: + rebuild_ocr_evidence(rgb, bad) + self.assertEqual(caught.exception.code, "HASH_MISMATCH") + with self.assertRaises(CropError): + rebuild_ocr_evidence(rgb, {}) + modified = rgb.copy() + modified[0, 0, 0] ^= 255 + with self.assertRaises(CropError) as caught: + rebuild_ocr_evidence(modified, metadata) + self.assertEqual(caught.exception.code, "HASH_MISMATCH") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ai/test_dfine_adapter.py b/tests/ai/test_dfine_adapter.py new file mode 100644 index 0000000..9a71ea6 --- /dev/null +++ b/tests/ai/test_dfine_adapter.py @@ -0,0 +1,187 @@ +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from apps.ai_inference.adapters.dfine import ( + ADAPTER_ID, + CLASS_COUNT, + CLASSES, + QUERY_COUNT, + AdapterError, + OnnxDetector, + adapt_outputs, + inspect_artifacts, + postprocess_project, + require_activation, + validate_session_contract, +) + + +class _Value: + def __init__(self, name, shape, dtype): + self.name, self.shape, self.type = name, shape, dtype + + +class _Session: + def get_inputs(self): + return [ + _Value("images", (1, 3, 640, 640), "tensor(float)"), + _Value("orig_target_sizes", (1, 2), "tensor(int64)"), + ] + + def get_outputs(self): + return [ + _Value("labels", (1, 300), "tensor(int64)"), + _Value("boxes", (1, 300, 4), "tensor(float)"), + _Value("scores", (1, 300), "tensor(float)"), + ] + + +class _Array: + def __init__(self, data, shape, dtype="float32"): + self.data, self.shape, self.dtype = data, shape, dtype + + def tolist(self): + return self.data + + +class DfineAdapterTests(unittest.TestCase): + def test_official_metadata_is_activation_eligible_and_keeps_80_labels(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "model.onnx").write_bytes(b"onnx-placeholder") + (root / "config.json").write_text( + json.dumps( + { + "model_type": "d_fine", + "num_queries": 300, + "id2label": {str(i): label for i, label in enumerate(CLASSES)}, + } + ), + encoding="utf-8", + ) + (root / "preprocessor_config.json").write_text( + json.dumps( + { + "size": {"height": 640, "width": 640}, + "do_resize": True, + "do_rescale": True, + "do_pad": False, + "do_normalize": False, + "rescale_factor": 1 / 255, + } + ), + encoding="utf-8", + ) + evidence = inspect_artifacts( + root / "model.onnx", root / "config.json", root / "preprocessor_config.json" + ) + require_activation(evidence) + self.assertTrue(evidence.activation_allowed) + self.assertEqual(ADAPTER_ID, "dfine-coco80-rgb-stretch-v1") + + def test_postprocess_and_adapt_accept_class_79_and_reject_80(self): + logits = [[-20.0] * CLASS_COUNT for _ in range(QUERY_COUNT)] + logits[0][79] = 8.0 + boxes = [[0.5, 0.5, 0.5, 0.5] for _ in range(QUERY_COUNT)] + rows = postprocess_project([logits], [boxes], [100, 200], 0.5) + self.assertEqual(rows[0]["class_id"], 79) + self.assertEqual(rows[0]["type"], CLASSES[79]) + labels = [79] + [0] * (QUERY_COUNT - 1) + scores = [0.9] + [0.0] * (QUERY_COUNT - 1) + boxes_xyxy = [[0, 0, 100, 200] for _ in range(QUERY_COUNT)] + self.assertEqual( + adapt_outputs([labels], [boxes_xyxy], [scores], [100, 200], 0.5)[0]["class_id"], 79 + ) + labels[-1] = 80 + with self.assertRaises(AdapterError) as error: + adapt_outputs([labels], [boxes_xyxy], [scores], [100, 200], 0.5) + self.assertEqual(error.exception.code, "MODEL_ERROR") + + def test_session_signature_is_fail_closed(self): + validate_session_contract(_Session()) + session = _Session() + session.get_inputs = lambda: [ + _Value("images", (1, 3, 320, 320), "tensor(float)"), + _Value("orig_target_sizes", (1, 2), "tensor(int64)"), + ] + with self.assertRaises(AdapterError): + validate_session_contract(session) + + def test_raw_export_dispatch_and_signature(self): + session = _Session() + session.get_inputs = lambda: [ + _Value("pixel_values", ("batch_size", 3, "height", "width"), "tensor(float)") + ] + session.get_outputs = lambda: [ + _Value("logits", ("batch_size", 300, 80), "tensor(float)"), + _Value("pred_boxes", ("batch_size", 300, 4), "tensor(float)"), + ] + self.assertEqual(validate_session_contract(session), "raw") + logits = [[-20.0] * 80 for _ in range(300)] + logits[0][39] = 8.0 + + def run(names, feed): + self.assertEqual(names, ["logits", "pred_boxes"]) + self.assertEqual(feed, {"pixel_values": "tensor"}) + return [ + _Array([logits], (1, 300, 80)), + _Array([[[0.5, 0.5, 0.5, 0.5]] * 300], (1, 300, 4)), + ] + + session.run = run + with patch( + "apps.ai_inference.adapters.dfine.preprocess_rgb", return_value=("tensor", None) + ): + result = OnnxDetector(session).detect(SimpleNamespace(size=(800, 400))) + self.assertEqual(result[0]["type"], "bottle") + self.assertEqual(result[0]["bbox"], [0.25, 0.25, 0.75, 0.75]) + session.get_inputs = lambda: [_Value("pixel_values", (2, 3, 640, 640), "tensor(float)")] + with self.assertRaises(AdapterError): + validate_session_contract(session) + + def test_low_score_degenerate_box_is_filtered_but_nonfinite_and_retained_invalid_fail(self): + logits = [[-20.0] * 80 for _ in range(300)] + boxes = [[0.5, 0.5, 0.5, 0.5] for _ in range(300)] + boxes[11] = [0.1, -0.05, 0.06, -0.005] + diagnostics = {} + self.assertEqual( + postprocess_project([logits], [boxes], [800, 400], 0.4, diagnostics=diagnostics), + [], + ) + self.assertEqual(diagnostics["below_threshold"], 300) + boxes[11][0] = float("nan") + with self.assertRaises(AdapterError): + postprocess_project([logits], [boxes], [800, 400], 0.4) + boxes[11][0] = 0.1 + logits[11][39] = 8.0 + with self.assertRaises(AdapterError): + postprocess_project([logits], [boxes], [800, 400], 0.4) + + def test_postprocessed_threshold_geometry_matches_raw_contract(self): + labels = [[39] * 300] + boxes = [[[0, 0, 1, -1]] * 300] + scores = [[0.01] * 300] + self.assertEqual(adapt_outputs(labels, boxes, scores, [800, 400], 0.4), []) + scores[0][0] = 0.4 + with self.assertRaises(AdapterError): + adapt_outputs(labels, boxes, scores, [800, 400], 0.4) + + def test_runtime_tensor_dtype_is_checked_before_casting(self): + session = _Session() + session.run = lambda *args: [ + _Array([[0] * 300], (1, 300), "float32"), + _Array([[[0, 0, 10, 10]] * 300], (1, 300, 4)), + _Array([[0.01] * 300], (1, 300)), + ] + with patch("apps.ai_inference.adapters.dfine.preprocess_rgb", return_value=(None, None)): + with self.assertRaises(AdapterError) as error: + OnnxDetector(session).detect(SimpleNamespace(size=(800, 400))) + self.assertEqual(error.exception.code, "SCHEMA_MISMATCH") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ai/test_ocrv6_adapter.py b/tests/ai/test_ocrv6_adapter.py new file mode 100644 index 0000000..964ab01 --- /dev/null +++ b/tests/ai/test_ocrv6_adapter.py @@ -0,0 +1,283 @@ +"""CPU OCR numerical, geometry, vocabulary and bounded execution checks.""" + +import importlib.util +import json +import multiprocessing +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch +from uuid import uuid4 + +from apps.ai_inference.adapters.dfine import AdapterError +from apps.ai_inference.adapters.ocrv6 import ( + crop_region, + db_regions, + decode_ctc, + det_tensor, + load_artifacts, + rec_tensor, + validate_session, +) +from apps.ai_inference.ocr_cpu import OcrOptions, _compute, run_cpu_ocr +from tests.ai.test_cpu_runner import crashed_child, delayed_child, successful_child + +REAL = all(importlib.util.find_spec(name) for name in ("numpy", "PIL", "cv2", "pyclipper")) + + +class OcrProcessTests(unittest.TestCase): + def test_ocr_timeout_and_crash_reclaimed_and_next_run_succeeds(self): + before = {child.pid for child in multiprocessing.active_children()} + for target, code in ((delayed_child, "AI_TIMEOUT"), (crashed_child, "MODEL_ERROR")): + with ( + patch.dict(os.environ, APP_ENV="test"), + patch("apps.ai_inference.ocr_cpu._child", target), + ): + with self.assertRaises(AdapterError) as error: + run_cpu_ocr(OcrOptions("det", "rec", timeout_seconds=1), ["x"], str(uuid4())) + self.assertEqual(error.exception.code, code) + self.assertEqual({child.pid for child in multiprocessing.active_children()}, before) + with ( + patch.dict(os.environ, APP_ENV="test"), + patch("apps.ai_inference.ocr_cpu._child", successful_child), + ): + self.assertEqual( + run_cpu_ocr(OcrOptions("d", "r"), ["x"], str(uuid4())), {"complete": True} + ) + + def test_production_refused_without_output_or_path_leak(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output.json" + private = Path(directory) / "private.png" + result = subprocess.run( + [ + sys.executable, + "-m", + "apps.ai_inference.ocr", + "--det-dir", + "det", + "--rec-dir", + "rec", + "--input", + str(private), + "--output", + str(output), + ], + env={**os.environ, "APP_ENV": "production"}, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(json.loads(result.stderr)["error"]["code"], "VALIDATION_ERROR") + self.assertNotIn(str(private), result.stderr) + self.assertFalse(output.exists()) + + def test_invalid_threshold_threads_deadline_and_image_count(self): + with patch.dict(os.environ, APP_ENV="test"): + for options in ( + OcrOptions("d", "r", text_min=float("nan")), + OcrOptions("d", "r", text_min=True), + OcrOptions("d", "r", threads=0), + OcrOptions("d", "r", timeout_seconds=181), + ): + with self.assertRaises(AdapterError): + run_cpu_ocr(options, ["x"], str(uuid4())) + with self.assertRaises(AdapterError): + run_cpu_ocr(OcrOptions("d", "r"), ["x"] * 4, str(uuid4())) + + +@unittest.skipUnless(REAL, "OCR numerical tests require the independent real CPU environment") +class OcrNumericalTests(unittest.TestCase): + def test_ctc_blank_duplicates_space_and_unicode(self): + import numpy as np + + characters = ("", "乙", "醇", " ") + indices = [0, 1, 1, 0, 1, 2, 2, 3, 3, 0] + probs = np.zeros((1, len(indices), len(characters)), np.float32) + for step, index in enumerate(indices): + probs[0, step, index] = 0.9 + probs[0, step, (index + 1) % 4] = 0.1 + text, confidence = decode_ctc(probs, characters) + self.assertEqual(text, "乙乙醇 ") + self.assertAlmostEqual(confidence, 0.9, places=6) + blank = np.zeros((1, 4, 4), np.float32) + blank[:, :, 0] = 1 + self.assertEqual(decode_ctc(blank, characters), ("", 0)) + + def test_ctc_nonfinite_bad_dtype_shape_and_probability_rejected(self): + import numpy as np + + valid = np.zeros((1, 4, 3), np.float32) + valid[:, :, 0] = 1 + cases = [ + valid.astype(np.float64), + valid[:, :, :2], + valid * 0.5, + np.full_like(valid, np.nan), + np.full_like(valid, np.inf), + valid * 2, + ] + for probs in cases: + with self.subTest(shape=probs.shape), self.assertRaises(AdapterError): + decode_ctc(probs, ("", "a", " ")) + + def test_bgr_normalization_padding_and_rec_width_limit(self): + import numpy as np + + rgb = np.full((32, 64, 3), [255, 0, 0], np.uint8) + det = det_tensor(rgb) + self.assertEqual(det.shape, (1, 3, 32, 64)) + self.assertEqual(det.dtype, np.float32) + self.assertTrue(det.flags.c_contiguous) + np.testing.assert_allclose( + det[0, :, 0, 0], [-0.485 / 0.229, -0.456 / 0.224, (1 - 0.406) / 0.225], atol=1e-6 + ) + rec = rec_tensor(rgb) + self.assertEqual(rec.shape, (1, 3, 48, 320)) + np.testing.assert_array_equal(rec[0, :, 0, 0], [-1, -1, 1]) + self.assertTrue((rec[0, :, :, 96:] == 0).all()) + wide = rec_tensor(np.zeros((48, 333, 3), np.uint8)) + self.assertEqual(wide.shape[-1], 336) + self.assertTrue((wide[0, :, :, 333:] == 0).all()) + with self.assertRaises(AdapterError): + rec_tensor(np.zeros((2, 200, 3), np.uint8)) + + def test_db_non_square_quad_bounds_sort_and_blank(self): + import numpy as np + + probs = np.zeros((1, 1, 96, 192), np.float32) + self.assertEqual(db_regions(probs, (384, 192)), []) + probs[0, 0, 10:20, 15:70] = 0.9 + probs[0, 0, 60:70, 95:170] = 0.8 + rows = db_regions(probs, (384, 192)) + self.assertEqual(len(rows), 2) + self.assertLess(rows[0]["quad_pixels"][0, 1], rows[1]["quad_pixels"][0, 1]) + for row in rows: + quad = row["quad_pixels"] + self.assertTrue((quad >= 0).all()) + self.assertTrue((quad[:, 0] <= 383).all()) + self.assertTrue((quad[:, 1] <= 191).all()) + self.assertLess(quad[0, 0], quad[1, 0]) + self.assertLess(quad[0, 1], quad[3, 1]) + with self.assertRaises(AdapterError): + db_regions(np.full_like(probs, np.nan), (384, 192)) + + def test_db_capacity_is_error_not_truncation(self): + import numpy as np + + probs = np.zeros((1, 1, 240, 400), np.float32) + for index in range(101): + y, x = 5 + index // 20 * 30, 5 + index % 20 * 20 + probs[0, 0, y : y + 6, x : x + 6] = 1 + with self.assertRaises(AdapterError) as error: + db_regions(probs, (400, 240)) + self.assertEqual(error.exception.code, "MODEL_ERROR") + probs[0, 0, y : y + 6, x : x + 6] = 0 + self.assertEqual(len(db_regions(probs, (400, 240))), 100) + + def test_perspective_crop_and_vertical_rotation_preserve_pixels(self): + import numpy as np + + rgb = np.full((80, 160, 3), [250, 10, 3], np.uint8) + quad = np.array([[10, 10], [110, 10], [110, 40], [10, 40]], np.float32) + crop = crop_region(rgb, quad) + self.assertEqual(crop.shape, (30, 100, 3)) + np.testing.assert_array_equal(crop[15, 50], [250, 10, 3]) + tall = np.array([[10, 10], [20, 10], [20, 60], [10, 60]], np.float32) + self.assertEqual(crop_region(rgb, tall).shape, (10, 50, 3)) + with self.assertRaises(AdapterError): + crop_region(rgb, np.zeros((4, 2), np.float32)) + for invalid in ( + quad[[0, 2, 1, 3]], + np.array([[10, 10], [10, 10], [110, 40], [10, 40]], np.float32), + np.full((4, 2), np.nan, np.float32), + ): + with self.assertRaises(AdapterError): + crop_region(rgb, invalid) + + def test_artifact_hash_tampering_rejected(self): + with tempfile.TemporaryDirectory() as directory: + (Path(directory) / "inference.onnx").write_bytes(b"untrusted") + with self.assertRaises(AdapterError) as error: + load_artifacts(directory, "det") + self.assertEqual(error.exception.code, "HASH_MISMATCH") + + def test_wrong_signature_or_provider_rejected(self): + from types import SimpleNamespace + + class Session: + output = SimpleNamespace( + name="fetch_name_0", type="tensor(float)", shape=["b", "t", 18710] + ) + providers = ["CPUExecutionProvider"] + + def get_inputs(self): + return [SimpleNamespace(name="x", type="tensor(float)", shape=["b", 3, 48, "w"])] + + def get_outputs(self): + return [self.output] + + def get_providers(self): + return self.providers + + session = Session() + validate_session(session, "rec") + session.providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + with self.assertRaises(AdapterError): + validate_session(session, "rec") + session.providers = ["CPUExecutionProvider"] + session.output.shape = ["b", "t", 18709] + with self.assertRaises(AdapterError): + validate_session(session, "rec") + + def test_report_ids_repeat_low_confidence_kept_no_chemical_facts_and_batch_capacity(self): + from PIL import Image + + class OCR: + count = 1 + + def smoke(self): + pass + + def recognize(self, image): + return [ + { + "text": "乙醇", + "confidence": 0.3, + "detection_confidence": 0.8, + "quad": [[0, 0], [1, 0], [1, 1], [0, 1]], + "crop_evidence": {}, + } + for _ in range(self.count) + ] + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "image.png" + with Image.new("RGB", (200, 100)) as image: + image.save(path) + ocr, run_id = OCR(), str(uuid4()) + with patch( + "apps.ai_inference.ocr_cpu.OnnxOCR.from_directories", return_value=ocr + ) as load: + first = _compute(OcrOptions("d", "r"), [path], run_id) + second = _compute(OcrOptions("d", "r"), [path], run_id) + self.assertEqual(first["images"][0]["lines"], second["images"][0]["lines"]) + row = first["images"][0]["lines"][0] + self.assertTrue(row["uncertain"]) + self.assertIsNone(row["parent_detection_id"]) + self.assertFalse(first["business_capabilities"]["chemical_entities"]) + ocr.count = 50 + report = _compute(OcrOptions("d", "r"), [path, path], run_id) + self.assertEqual(sum(len(i["lines"]) for i in report["images"]), 100) + ocr.count = 51 + with self.assertRaises(AdapterError): + _compute(OcrOptions("d", "r"), [path, path], run_id) + self.assertEqual(load.call_count, 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ai/test_pipeline_cpu.py b/tests/ai/test_pipeline_cpu.py new file mode 100644 index 0000000..cd9a4a1 --- /dev/null +++ b/tests/ai/test_pipeline_cpu.py @@ -0,0 +1,284 @@ +"""Whole-batch quality gate, decode ownership, capacities and bounded joint CPU runs.""" + +import importlib.util +import json +import multiprocessing +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch +from uuid import uuid4 + +from apps.ai_inference.adapters.dfine import AdapterError +from apps.ai_inference.adapters.quality import QualityThresholds +from apps.ai_inference.cpu import _decode_local +from apps.ai_inference.pipeline_cpu import PipelineOptions, _compute, run_cpu_pipeline +from tests.ai.test_cpu_runner import crashed_child, delayed_child, successful_child + +REAL = all(importlib.util.find_spec(name) for name in ("numpy", "PIL", "cv2", "pyclipper")) + + +class PipelineProcessTests(unittest.TestCase): + def test_timeout_and_crash_reclaimed_and_next_run_succeeds(self): + before = {child.pid for child in multiprocessing.active_children()} + for target, code in ((delayed_child, "AI_TIMEOUT"), (crashed_child, "MODEL_ERROR")): + with ( + patch.dict(os.environ, APP_ENV="test"), + patch("apps.ai_inference.pipeline_cpu._child", target), + ): + with self.assertRaises(AdapterError) as error: + run_cpu_pipeline( + PipelineOptions(quality_only=True, timeout_seconds=1), + ["unused"], + str(uuid4()), + ) + self.assertEqual(error.exception.code, code) + self.assertEqual({child.pid for child in multiprocessing.active_children()}, before) + with ( + patch.dict(os.environ, APP_ENV="test"), + patch("apps.ai_inference.pipeline_cpu._child", successful_child), + ): + self.assertEqual( + run_cpu_pipeline(PipelineOptions(quality_only=True), ["x"], str(uuid4())), + {"complete": True}, + ) + + def test_invalid_configuration_fails_before_spawn(self): + for options in ( + PipelineOptions(), + PipelineOptions(quality_only="true"), + PipelineOptions(quality_only=True, quality=QualityThresholds(blur_min=True)), + PipelineOptions(quality_only=True, detection_min=float("nan")), + PipelineOptions(quality_only=True, text_min=2), + PipelineOptions(quality_only=True, timeout_seconds=181), + ): + with patch.dict(os.environ, APP_ENV="test"), self.assertRaises(AdapterError): + run_cpu_pipeline(options, ["x"], str(uuid4())) + + def test_cli_production_refusal_no_path_leak_or_output(self): + with tempfile.TemporaryDirectory() as directory: + path, output = Path(directory) / "private.png", Path(directory) / "out.json" + result = subprocess.run( + [ + sys.executable, + "-m", + "apps.ai_inference.pipeline", + "--quality-only", + "--input", + str(path), + "--output", + str(output), + ], + env={**os.environ, "APP_ENV": "production"}, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 1) + self.assertEqual(json.loads(result.stderr)["error"]["code"], "VALIDATION_ERROR") + self.assertNotIn(str(path), result.stderr) + self.assertFalse(output.exists()) + + +@unittest.skipUnless(REAL, "Joint pixel tests require the independent real CPU environment") +class PipelinePixelTests(unittest.TestCase): + def setUp(self): + import numpy as np + from PIL import Image + + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.paths = [Path(self.directory.name) / f"{i}.png" for i in range(3)] + for path in self.paths: + pixels = np.random.default_rng(34).integers(15, 220, (64, 128, 3), np.uint8) + with Image.fromarray(pixels) as image: + image.save(path) + self.options = PipelineOptions("model", "det", "rec") + self.run_id = str(uuid4()) + + def test_one_failed_image_gates_whole_batch_and_loads_no_models(self): + from PIL import Image + + with Image.new("RGB", (64, 32)) as image: + image.save(self.paths[1]) + with ( + patch("apps.ai_inference.pipeline_cpu.OnnxDetector.from_path") as det, + patch("apps.ai_inference.pipeline_cpu.OnnxOCR.from_directories") as ocr, + ): + report = _compute(self.options, self.paths, self.run_id) + det.assert_not_called() + ocr.assert_not_called() + self.assertEqual(report["outcome"], "needs_retake") + self.assertFalse(report["models_executed"]) + self.assertTrue(report["images"][0]["quality"]["passed"]) + self.assertEqual(report["images"][1]["quality"]["reasons"], ["blur", "dark"]) + self.assertEqual(report["artifacts"], {}) + self.assertIsNone(report["runtime"]["detector"]) + for row in report["images"]: + self.assertFalse(row["inference_executed"]) + self.assertEqual(row["lines"], []) + self.assertEqual(row["detections"], []) + + def test_quality_only_success_is_not_business_completion(self): + report = _compute(PipelineOptions(quality_only=True), self.paths, self.run_id) + self.assertEqual(report["outcome"], "quality_passed") + self.assertTrue(report["review_required"]) + self.assertFalse(report["models_executed"]) + self.assertIsNone(report["runtime"]["ocr"]) + self.assertFalse(any(report["business_capabilities"].values())) + + def _models(self, detection_count=1, line_count=1, fail_second=False): + class Detector: + providers = ("CPUExecutionProvider",) + evidence = SimpleNamespace( + model_sha256="a" * 64, config_sha256="b" * 64, preprocessor_sha256="c" * 64 + ) + inputs = [] + + def detect(self, image, threshold, *, diagnostics=None): + if diagnostics is not None: + self.inputs.append(image) + if fail_second and len(self.inputs) == 2: + raise AdapterError("MODEL_ERROR", "Test output failure") + diagnostics.update(candidates=300, kept=detection_count) + return [ + {"class_id": 39, "type": "bottle", "bbox": [0, 0, 1, 1], "confidence": 0.8} + for _ in range(detection_count) + ] + + class OCR: + inputs = [] + + def smoke(self): + pass + + def recognize(self, image): + self.inputs.append(image) + return [ + { + "text": "乙醇", + "confidence": 0.3, + "detection_confidence": 0.8, + "quad": [[0, 0], [1, 0], [1, 1], [0, 1]], + "crop_evidence": {}, + } + for _ in range(line_count) + ] + + return Detector(), OCR() + + def test_sessions_loaded_once_decode_once_shared_pixels_ids_and_cleanup(self): + det, ocr = self._models() + with ( + patch( + "apps.ai_inference.pipeline_cpu.OnnxDetector.from_path", return_value=det + ) as load_det, + patch( + "apps.ai_inference.pipeline_cpu.OnnxOCR.from_directories", return_value=ocr + ) as load_ocr, + patch("apps.ai_inference.pipeline_cpu._decode_local", wraps=_decode_local) as decode, + ): + report = _compute(self.options, self.paths, self.run_id) + load_det.assert_called_once() + load_ocr.assert_called_once() + self.assertEqual(decode.call_count, 3) + self.assertEqual(det.inputs, ocr.inputs) + self.assertEqual(report["outcome"], "needs_review") + self.assertTrue(report["models_executed"]) + self.assertFalse(any(report["business_capabilities"].values())) + for image, row in zip(det.inputs, report["images"]): + self.assertTrue(row["inference_executed"]) + self.assertTrue(row["lines"][0]["uncertain"]) + self.assertIsNone(row["lines"][0]["parent_detection_id"]) + self.assertIsNone(row["detections"][0]["parent_detection_id"]) + with self.assertRaises(ValueError): + image.getpixel((0, 0)) + again = _compute(self.options, self.paths, self.run_id) + for left, right in zip(report["images"], again["images"]): + self.assertEqual(left["image_id"], right["image_id"]) + self.assertEqual(left["detections"], right["detections"]) + self.assertEqual(left["lines"], right["lines"]) + + def test_both_batch_capacities_fail_without_truncation_and_100_is_allowed(self): + for detections, lines, valid in ((50, 50, True), (51, 1, False), (1, 51, False)): + det, ocr = self._models(detections, lines) + with ( + patch("apps.ai_inference.pipeline_cpu.OnnxDetector.from_path", return_value=det), + patch("apps.ai_inference.pipeline_cpu.OnnxOCR.from_directories", return_value=ocr), + ): + if valid: + report = _compute(self.options, self.paths[:2], self.run_id) + self.assertEqual(sum(len(i["detections"]) for i in report["images"]), 100) + self.assertEqual(sum(len(i["lines"]) for i in report["images"]), 100) + else: + with self.assertRaises(AdapterError) as error: + _compute(self.options, self.paths[:2], self.run_id) + self.assertEqual(error.exception.code, "MODEL_ERROR") + + def test_later_model_failure_closes_all_inputs_and_never_returns_partial_success(self): + det, ocr = self._models(fail_second=True) + with ( + patch("apps.ai_inference.pipeline_cpu.OnnxDetector.from_path", return_value=det), + patch("apps.ai_inference.pipeline_cpu.OnnxOCR.from_directories", return_value=ocr), + ): + with self.assertRaises(AdapterError): + _compute(self.options, self.paths, self.run_id) + self.assertEqual(len(ocr.inputs), 1) + for image in det.inputs: + with self.assertRaises(ValueError): + image.getpixel((0, 0)) + + def test_decode_failure_reclaims_previous_images_without_loading_models(self): + self.paths[1].write_bytes(b"not an image") + decoded = [] + + def decode(path): + result = _decode_local(path) + decoded.append(result[0]) + return result + + with ( + patch("apps.ai_inference.pipeline_cpu._decode_local", side_effect=decode), + patch("apps.ai_inference.pipeline_cpu.OnnxDetector.from_path") as load, + ): + with self.assertRaises(AdapterError): + _compute(self.options, self.paths, self.run_id) + load.assert_not_called() + for image in decoded: + with self.assertRaises(ValueError): + image.getpixel((0, 0)) + + def test_real_quality_cli_output_and_existing_file_preservation(self): + output = Path(self.directory.name) / "out.json" + command = [ + sys.executable, + "-m", + "apps.ai_inference.pipeline", + "--quality-only", + "--input", + str(self.paths[0]), + "--output", + str(output), + ] + first = subprocess.run( + command, env={**os.environ, "APP_ENV": "test"}, capture_output=True, timeout=15 + ) + self.assertEqual(first.returncode, 0, first.stderr) + raw = output.read_bytes() + report = json.loads(raw) + self.assertEqual(report["outcome"], "quality_passed") + self.assertFalse(report["models_executed"]) + second = subprocess.run( + command, env={**os.environ, "APP_ENV": "test"}, capture_output=True, timeout=15 + ) + self.assertEqual(second.returncode, 1) + self.assertEqual(json.loads(second.stderr)["error"]["code"], "OUTPUT_UNAVAILABLE") + self.assertEqual(output.read_bytes(), raw) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ai/test_quality_pixels.py b/tests/ai/test_quality_pixels.py new file mode 100644 index 0000000..b20637b --- /dev/null +++ b/tests/ai/test_quality_pixels.py @@ -0,0 +1,133 @@ +"""Real pixel goldens and independent kernel checks for quality-rgb-lap1-v1.""" + +import importlib.util +import unittest + +from apps.ai_inference.adapters.dfine import AdapterError +from apps.ai_inference.adapters.quality import QualityThresholds, evaluate_quality, resize_shape + +REAL = all(importlib.util.find_spec(name) for name in ("numpy", "PIL")) + + +class QualityConfigTests(unittest.TestCase): + def test_resize_integer_rounding_and_threshold_validation(self): + self.assertEqual(resize_shape(1025, 513), (1024, 512)) + self.assertEqual(resize_shape(513, 1025), (512, 1024)) + self.assertEqual(resize_shape(7, 3), (7, 3)) + self.assertEqual(resize_shape(1, 10000), (1, 1024)) + for dimensions in ((0, 1), (True, 1), (1.0, 2)): + with self.assertRaises(AdapterError): + resize_shape(*dimensions) + for limits in ( + QualityThresholds(blur_min=True), + QualityThresholds(blur_min=-1), + QualityThresholds(dark_min=float("nan")), + QualityThresholds(glare_max=float("inf")), + QualityThresholds(dark_min=1.1), + QualityThresholds(glare_max=-0.1), + ): + with self.assertRaises(AdapterError): + limits.validate() + + +@unittest.skipUnless(REAL, "Quality pixel tests require the independent real CPU environment") +class QualityPixelTests(unittest.TestCase): + def test_black_white_and_rgb_gray_goldens(self): + from PIL import Image + + for color, brightness, glare, reasons in ( + ("black", 0, 0, ["blur", "dark"]), + ("white", 1, 1, ["blur", "glare"]), + ((255, 0, 0), 77 / 255, 0, ["blur"]), + ((0, 255, 0), 149 / 255, 0, ["blur"]), + ((0, 0, 255), 29 / 255, 0, ["blur", "dark"]), + ): + with self.subTest(color=color), Image.new("RGB", (1, 1), color) as image: + scores = evaluate_quality(image) + self.assertEqual(scores["blur_score"], 0) + self.assertEqual(scores["brightness"], brightness) + self.assertEqual(scores["glare_ratio"], glare) + self.assertEqual(scores["reasons"], reasons) + self.assertFalse(scores["passed"]) + + def test_single_row_and_column_reflect101_population_variance(self): + import numpy as np + from PIL import Image + + pixels = np.array([[[0, 0, 0], [255, 255, 255]]], np.uint8) + for array in (pixels, pixels.transpose(1, 0, 2)): + with Image.fromarray(array) as image: + scores = evaluate_quality(image) + self.assertEqual(scores["blur_score"], 260100) + self.assertEqual(scores["brightness"], 0.5) + self.assertEqual(scores["glare_ratio"], 0.5) + self.assertEqual(scores["reasons"], ["glare"]) + + def test_threshold_equality_and_no_score_rounding(self): + import math + + import numpy as np + from PIL import Image + + with Image.fromarray(np.array([[[0] * 3, [255] * 3]], np.uint8)) as image: + self.assertTrue(evaluate_quality(image, QualityThresholds(260100, 0.5, 0.5))["passed"]) + result = evaluate_quality( + image, + QualityThresholds( + math.nextafter(260100, math.inf), + math.nextafter(0.5, math.inf), + math.nextafter(0.5, -math.inf), + ), + ) + self.assertEqual(result["reasons"], ["blur", "dark", "glare"]) + with Image.new("RGB", (7, 3), (255, 0, 0)) as image: + result = evaluate_quality(image) + self.assertEqual(result["brightness"], 77 / 255) + self.assertNotEqual(result["brightness"], round(77 / 255, 4)) + + def test_real_pillow_resize_is_bilinear_and_input_remains_open(self): + import numpy as np + from PIL import Image + + from tools.design.readiness_reference import quality_resized + + pixels = np.random.default_rng(34).integers(0, 256, (1, 1025, 3), np.uint8) + with ( + Image.fromarray(pixels) as image, + image.resize((1024, 1), Image.Resampling.BILINEAR) as expected, + ): + actual = evaluate_quality(image) + reference = quality_resized(np.asarray(expected).tolist()) + self.assertEqual(actual["analysis_size"], [1024, 1]) + for key in reference: + self.assertAlmostEqual(actual[key], reference[key], places=8) + self.assertEqual(image.size, (1025, 1)) + self.assertEqual(image.getpixel((0, 0)), tuple(pixels[0, 0])) + + @unittest.skipUnless(importlib.util.find_spec("cv2"), "Independent OpenCV kernel comparison") + def test_numpy_kernel_matches_actual_opencv_on_borders_and_singletons(self): + import cv2 + import numpy as np + from PIL import Image + + for shape in ((1, 1, 3), (1, 7, 3), (9, 1, 3), (11, 13, 3)): + pixels = np.random.default_rng(34).integers(0, 256, shape, np.uint8) + rgb = pixels.astype(np.uint32) + gray = (77 * rgb[:, :, 0] + 150 * rgb[:, :, 1] + 29 * rgb[:, :, 2] + 128) // 256 + lap = cv2.Laplacian( + gray.astype(np.float64), cv2.CV_64F, ksize=1, borderType=cv2.BORDER_REFLECT_101 + ) + with Image.fromarray(pixels) as image: + actual = evaluate_quality(image) + self.assertEqual(actual["blur_score"], float(lap.var(ddof=0))) + + def test_non_rgb_is_rejected_without_silent_conversion(self): + from PIL import Image + + for mode in ("L", "RGBA"): + with Image.new(mode, (8, 8)) as image, self.assertRaises(AdapterError): + evaluate_quality(image) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/design/dfine_reference.py b/tools/design/dfine_reference.py index ea9d86b..1cbd521 100644 --- a/tools/design/dfine_reference.py +++ b/tools/design/dfine_reference.py @@ -1,84 +1,298 @@ """Pure D-FINE contract reference; no image library, model, GPU or business writes.""" + import math -ADAPTER_ID = 'dfine-n4-rgb-stretch-v1' -SOURCE_COMMIT = '956d1709314c2c6a4df6f34de232054578a7449f' -CLASSES = ('bottle', 'label', 'shelf', 'cabinet') -CATEGORY_TO_CLASS = {1: 0, 2: 1, 3: 2, 4: 3} +ADAPTER_ID = "dfine-coco80-rgb-stretch-v1" +SOURCE_COMMIT = "956d1709314c2c6a4df6f34de232054578a7449f" +CLASSES = ( + "person", + "bicycle", + "car", + "motorbike", + "aeroplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "sofa", + "pottedplant", + "bed", + "diningtable", + "toilet", + "tvmonitor", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", +) +# Standard COCO category IDs are sparse; model class IDs are contiguous 0..79. +COCO_CATEGORY_IDS = ( + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 27, + 28, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 67, + 70, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 84, + 85, + 86, + 87, + 88, + 89, + 90, +) +CATEGORY_TO_CLASS = {category: index for index, category in enumerate(COCO_CATEGORY_IDS)} CLASS_TO_CATEGORY = {value: key for key, value in CATEGORY_TO_CLASS.items()} QUERY_COUNT = 300 MAX_DETECTIONS = 100 + def require(ok, message): if not ok: raise ValueError(message) + def category_to_class(category_id): - require(type(category_id) is int and category_id in CATEGORY_TO_CLASS, 'invalid COCO category') + require(type(category_id) is int and category_id in CATEGORY_TO_CLASS, "invalid COCO category") return CATEGORY_TO_CLASS[category_id] + def class_to_category(class_id): - require(type(class_id) is int and class_id in CLASS_TO_CATEGORY, 'invalid model class') + require(type(class_id) is int and class_id in CLASS_TO_CATEGORY, "invalid model class") return CLASS_TO_CATEGORY[class_id] + def finite(value): return type(value) in (int, float) and math.isfinite(value) + def image_size(size): - require(len(size) == 2 and all(type(x) is int and x > 0 for x in size), 'size must be [width,height]') + require( + len(size) == 2 and all(type(x) is int and x > 0 for x in size), + "size must be [width,height]", + ) return size + def rgb_pixel_to_float(pixel): - require(len(pixel) == 3 and all(type(x) is int and 0 <= x <= 255 for x in pixel), 'invalid RGB8 pixel') + require( + len(pixel) == 3 and all(type(x) is int and 0 <= x <= 255 for x in pixel), + "invalid RGB8 pixel", + ) return [x / 255.0 for x in pixel] + def query_outputs(logits, boxes_cxcywh, original_size): - """Mirror project export wrapper: qmax sigmoid, no TopK/NMS; query order retained. + """Mirror D-FINE export wrapper: qmax sigmoid, no TopK/NMS; query order retained. - Inputs are unbatched [300,4] normalized model boxes and logits. Export uses + Inputs are unbatched [300,4] normalized model boxes and [300,80] logits. Export uses float32 tensors; this scalar float64 reference specifies semantics, not GPU tolerance. """ width, height = image_size(original_size) - require(len(logits) == len(boxes_cxcywh) == QUERY_COUNT, 'SCHEMA_MISMATCH: query count') + require(len(logits) == len(boxes_cxcywh) == QUERY_COUNT, "SCHEMA_MISMATCH: query count") labels, boxes, scores = [], [], [] for row, box in zip(logits, boxes_cxcywh): - require(len(row) == len(box) == 4, 'SCHEMA_MISMATCH: four classes/box coordinates') - require(all(finite(x) for x in row + box), 'MODEL_ERROR: nonfinite output') - label = max(range(4), key=lambda i: (row[i], -i)) + require(len(row) == 80 and len(box) == 4, "SCHEMA_MISMATCH: 80 classes/box coordinates") + require(all(finite(x) for x in row + box), "MODEL_ERROR: nonfinite output") + label = max(range(80), key=lambda i: (row[i], -i)) value = row[label] - score = 1 / (1 + math.exp(-value)) if value >= 0 else math.exp(value) / (1 + math.exp(value)) + score = ( + 1 / (1 + math.exp(-value)) if value >= 0 else math.exp(value) / (1 + math.exp(value)) + ) cx, cy, bw, bh = box - require(bw > 0 and bh > 0, 'MODEL_ERROR: nonpositive box') labels.append(label) scores.append(score) - boxes.append([(cx-bw/2)*width, (cy-bh/2)*height, (cx+bw/2)*width, (cy+bh/2)*height]) + boxes.append( + [ + (cx - bw / 2) * width, + (cy - bh / 2) * height, + (cx + bw / 2) * width, + (cy + bh / 2) * height, + ] + ) return labels, boxes, scores + def adapt_outputs(labels, boxes_xyxy, scores, original_size, detection_min): """Validate all candidates before filtering; return image-normalized boxes.""" width, height = image_size(original_size) - require(finite(detection_min) and 0 <= detection_min <= 1, 'invalid threshold') - require(len(labels) == len(boxes_xyxy) == len(scores) == QUERY_COUNT, 'SCHEMA_MISMATCH: output shape') + require(finite(detection_min) and 0 <= detection_min <= 1, "invalid threshold") + require( + len(labels) == len(boxes_xyxy) == len(scores) == QUERY_COUNT, + "SCHEMA_MISMATCH: output shape", + ) result = [] for query_index, (label, box, score) in enumerate(zip(labels, boxes_xyxy, scores)): class_to_category(label) - require(len(box) == 4, 'SCHEMA_MISMATCH: box shape') - require(all(finite(x) for x in box) and finite(score), 'MODEL_ERROR: nonfinite output') - require(0 <= score <= 1 and box[2] > box[0] and box[3] > box[1], 'MODEL_ERROR: score/box') + require(len(box) == 4, "SCHEMA_MISMATCH: box shape") + require(all(finite(x) for x in box) and finite(score), "MODEL_ERROR: nonfinite output") + require(0 <= score <= 1, "MODEL_ERROR: score") if score < detection_min: continue + require(box[2] > box[0] and box[3] > box[1], "MODEL_ERROR: nonpositive retained box") x1, y1, x2, y2 = box - clipped = [max(0, min(width, x1))/width, max(0, min(height, y1))/height, - max(0, min(width, x2))/width, max(0, min(height, y2))/height] + clipped = [ + max(0, min(width, x1)) / width, + max(0, min(height, y1)) / height, + max(0, min(width, x2)) / width, + max(0, min(height, y2)) / height, + ] if clipped[2] <= clipped[0] or clipped[3] <= clipped[1]: continue - result.append({'class_id': label, 'type': CLASSES[label], 'bbox': clipped, - 'confidence': score, 'query_index': query_index}) - result.sort(key=lambda row: (row['bbox'][1], row['bbox'][0], row['bbox'][3], row['bbox'][2], - row['class_id'], row['confidence'], row['query_index'])) + result.append( + { + "class_id": label, + "type": CLASSES[label], + "bbox": clipped, + "confidence": score, + "query_index": query_index, + } + ) + result.sort( + key=lambda row: ( + row["bbox"][1], + row["bbox"][0], + row["bbox"][3], + row["bbox"][2], + row["class_id"], + row["confidence"], + row["query_index"], + ) + ) return result + def check_request_limit(image_results): - require(1 <= len(image_results) <= 3, 'invalid image count') - require(sum(map(len, image_results)) <= MAX_DETECTIONS, 'MODEL_ERROR: detection capacity exceeded') + require(1 <= len(image_results) <= 3, "invalid image count") + require( + sum(map(len, image_results)) <= MAX_DETECTIONS, "MODEL_ERROR: detection capacity exceeded" + ) return True diff --git a/tools/design/extra_specs.py b/tools/design/extra_specs.py index f9d00a4..988522a 100644 --- a/tools/design/extra_specs.py +++ b/tools/design/extra_specs.py @@ -1,5 +1,6 @@ """AI, rule, model and queue schemas share the public type conventions.""" import copy +from dfine_reference import CLASSES as DFINE_CLASSES def extend(g): globals().update(g) @@ -25,7 +26,7 @@ def extend(g): op('post','/rule-versions','importRuleVersion','RuleVersion','RuleImport','admin',201) model('ModelArtifact',{'role':enum('detector','ocr','quality','dictionary'),'object_key':text(1024),'sha256':SHA,'license':text(200),'runtime':text(100)}) model('Thresholds',{'detection_min':PROB,'ocr_min':PROB,'entity_min':PROB,'blur_min':{'type':'number','minimum':0},'dark_min':PROB,'glare_max':PROB,'adjacent_gap_ratio':{'type':'number','minimum':0,'maximum':2}}) - model('ModelManifest',{'bundle_id':ID,'version_label':text(64),'pipeline_version':enum('vision-v1'),'dictionary_version_id':ID,'dictionary_sha256':SHA,'git_commit':{'type':'string','pattern':'^[a-f0-9]{40}$'},'dataset_version':text(100),'device_profiles':arr(enum('cpu','cuda'),1,2),'artifacts':arr(ref('ModelArtifact'),4,4),'thresholds':ref('Thresholds'),'input_max_side':integer(512,2048),'detector_backend':enum('onnxruntime'),'ocr_backend':enum('paddleocr'),'evaluation_report_key':text(1024),'evaluation_passed':BOOL}) + model('ModelManifest',{'bundle_id':ID,'version_label':text(64),'pipeline_version':enum('vision-v1'),'dictionary_version_id':ID,'dictionary_sha256':SHA,'git_commit':{'type':'string','pattern':'^[a-f0-9]{40}$'},'dataset_version':text(100),'device_profiles':arr(enum('cpu','cuda'),1,2),'artifacts':arr(ref('ModelArtifact'),4,4),'thresholds':ref('Thresholds'),'input_max_side':integer(512,2048),'detector_backend':enum('onnxruntime'),'ocr_backend':enum('onnxruntime'),'evaluation_report_key':text(1024),'evaluation_passed':BOOL}) model('ModelImport',{'manifest':ref('ModelManifest'),'reason':text(2000)}) op('get','/model-versions/{id}/manifest','getModelManifest','ModelManifest',permission='admin') op('post','/model-versions','importModelVersion','ModelVersion','ModelImport','admin',201) @@ -38,7 +39,7 @@ def am(n,p,optional=()): A[n]=obj(p,optional);return ref(n) am('InferenceRequest',{'run_id':ID,'attempt_id':ID,'fencing_token':integer(1),'tenant_id':ID,'laboratory_id':ID,'item_id':ID,'submission_revision':integer(1),'model_bundle_id':ID,'model_checksum':SHA,'dictionary_version_id':ID,'dictionary_sha256':SHA,'pipeline_version':enum('vision-v1'),'device_profile':enum('cpu','cuda'),'deadline_at':DT,'request_hash':SHA,'image_refs':arr(ref('ImageRef'),1,3)}) am('Point',{'x':PROB,'y':PROB}) am('Quality',{'image_id':ID,'status':enum('pass','needs_retake'),'reasons':arr(enum('blur','dark','glare','occluded','unreadable'),hi=5),'blur_score':{'type':'number','minimum':0},'brightness':PROB,'glare_ratio':PROB}) - am('Detection',{'detection_id':ID,'image_id':ID,'parent_detection_id':nullable(ID),'type':enum('bottle','label','shelf','cabinet'),'bbox':arr(PROB,4,4),'confidence':PROB}) + am('Detection',{'detection_id':ID,'image_id':ID,'parent_detection_id':nullable(ID),'class_id':{'type':'integer','minimum':0,'maximum':79},'type':enum(*DFINE_CLASSES),'bbox':arr(PROB,4,4),'confidence':PROB}) am('CropRecipe',{'crop_id':ID,'image_id':ID,'detection_id':ID,'quad':arr(ref('Point'),4,4),'output_width':integer(1,2048),'output_height':integer(1,2048),'transform_version':enum('perspective-rgb-v1')}) am('OCRField',{'image_id':ID,'detection_id':ID,'crop_id':ID,'field':enum('name','expiry','production','opened','date_unknown','concentration','hazard_mark'),'raw_text':text(2000,0),'normalized_text':nullable(text(500)),'confidence':PROB}) am('Candidate',{'entity_id':ID,'canonical_name':text(),'confidence':PROB,'match_method':enum('cas_exact','alias_exact','fuzzy')}) diff --git a/tools/design/release_reference.py b/tools/design/release_reference.py index 81e6356..8789a06 100644 --- a/tools/design/release_reference.py +++ b/tools/design/release_reference.py @@ -49,10 +49,10 @@ def validate_manifest(manifest,environment,tenant_id=None,policy=None,report=Non require(manifest['purpose']=='development' and manifest['model_family']=='mock_fixture' and manifest['runtime_profile']=='fixture-v1','invalid fixture identity') require(manifest['detector_backend']=='mock' and manifest['ocr_backend']=='mock','invalid fixture backends') else: - require(manifest['adapter_id']=='dfine-n4-rgb-stretch-v1' and manifest['model_family']=='dfine_n','unsupported adapter') + require(manifest['adapter_id']=='dfine-coco80-rgb-stretch-v1' and manifest['model_family']=='dfine_n','unsupported adapter') require(manifest['source_ref']=='https://github.com/Peterande/D-FINE' and manifest['source_commit']==SOURCE_COMMIT,'unsupported source revision') - require(manifest['detector_backend']=='onnxruntime' and manifest['ocr_backend']=='paddleocr','unsupported backend') - expected={'dfine-cpu-fp32-ocrv4cpu-v1':['cpu'],'dfine-cuda-fp32-ocrv4cpu-v1':['cuda']} + require(manifest['detector_backend']=='onnxruntime' and manifest['ocr_backend']=='onnxruntime','unsupported backend') + expected={'dfine-cpu-fp32-ocrv6smallcpu-v1':['cpu'],'dfine-cuda-fp32-ocrv6smallcpu-v1':['cuda']} require(manifest['runtime_profile'] in expected and manifest['device_profiles']==expected[manifest['runtime_profile']],'device/runtime mismatch') if environment!='production':return True require(not mock and manifest['purpose']=='production','development bundle cannot run in production') diff --git a/tools/design/release_specs.py b/tools/design/release_specs.py index db24790..5bef629 100644 --- a/tools/design/release_specs.py +++ b/tools/design/release_specs.py @@ -4,15 +4,15 @@ def extend_release(g): S['Session']['properties']['environment']=enum('dev','test','production');S['Session']['required'].append('environment') S['InferenceRun']['properties']['is_simulated']=BOOL;S['InferenceRun']['required'].append('is_simulated') version=OUT['contracts/inference-v1.yaml']['components']['schemas']['Version'] - version['properties'].update({'purpose':enum('development','evaluation','production'),'is_simulated':BOOL,'adapter_id':enum('dfine-n4-rgb-stretch-v1','fixture-v1'),'runtime_profile':enum('dfine-cpu-fp32-ocrv4cpu-v1','dfine-cuda-fp32-ocrv4cpu-v1','fixture-v1'),'runtime_lock_sha256':SHA,'detector_device':enum('cpu','cuda','mock'),'ocr_device':enum('cpu','mock')}) + version['properties'].update({'purpose':enum('development','evaluation','production'),'is_simulated':BOOL,'adapter_id':enum('dfine-coco80-rgb-stretch-v1','fixture-v1'),'runtime_profile':enum('dfine-cpu-fp32-ocrv6smallcpu-v1','dfine-cuda-fp32-ocrv6smallcpu-v1','fixture-v1'),'runtime_lock_sha256':SHA,'detector_device':enum('cpu','cuda','mock'),'ocr_device':enum('cpu','mock')}) version['required']+=['purpose','is_simulated','adapter_id','runtime_profile','runtime_lock_sha256','detector_device','ocr_device'] model('Calibration',{'status':enum('uncalibrated','calibrated','failed'),'validation_split_sha256':nullable(SHA),'report_sha256':nullable(SHA)}) model('InferenceRuntime',{'detector_device':enum('cpu','cuda','mock'),'ocr_device':enum('cpu','mock'),'detector_precision':enum('fp32','mock'),'preprocess_id':enum('dfine-rgb-stretch640-v1','fixture-v1'),'postprocess_id':enum('dfine-qmax-v1','fixture-v1'),'onnx_opset':{'type':'integer','enum':[16,0]},'runtime_lock_sha256':SHA}) - additions={'purpose':enum('development','evaluation','production'),'adapter_id':enum('dfine-n4-rgb-stretch-v1','fixture-v1'),'model_family':enum('dfine_n','mock_fixture'),'source_ref':text(200),'source_commit':{'type':'string','pattern':'^[a-f0-9]{40}$'},'runtime_profile':enum('dfine-cpu-fp32-ocrv4cpu-v1','dfine-cuda-fp32-ocrv4cpu-v1','fixture-v1'),'runtime':ref('InferenceRuntime'),'content_sha256':SHA,'calibration':ref('Calibration'),'evaluation_policy_id':nullable(text(80)),'evaluation_policy_sha256':nullable(SHA),'evaluation_report_sha256':nullable(SHA)} + additions={'purpose':enum('development','evaluation','production'),'adapter_id':enum('dfine-coco80-rgb-stretch-v1','fixture-v1'),'model_family':enum('dfine_n','mock_fixture'),'source_ref':text(200),'source_commit':{'type':'string','pattern':'^[a-f0-9]{40}$'},'runtime_profile':enum('dfine-cpu-fp32-ocrv6smallcpu-v1','dfine-cuda-fp32-ocrv6smallcpu-v1','fixture-v1'),'runtime':ref('InferenceRuntime'),'content_sha256':SHA,'calibration':ref('Calibration'),'evaluation_policy_id':nullable(text(80)),'evaluation_policy_sha256':nullable(SHA),'evaluation_report_sha256':nullable(SHA)} S['ModelManifest']['properties'].update(additions);S['ModelManifest']['required']+=list(additions) S['ModelManifest']['properties']['input_max_side']={'type':'integer','const':640} S['ModelManifest']['properties']['detector_backend']=enum('onnxruntime','mock') - S['ModelManifest']['properties']['ocr_backend']=enum('paddleocr','mock') + S['ModelManifest']['properties']['ocr_backend']=enum('onnxruntime','mock') S['ModelManifest']['properties']['evaluation_report_key']=nullable(text(1024)) metric_ids=['DET-P','DET-R','OCR-NAME','OCR-DATE','ENT-P','ENT-COVER','ENT-OOV','Q-FALSE-REJECT','Q-FALSE-ACCEPT','RISK-P','RISK-R','UNCERTAIN-ROUTING','LATENCY-CUDA'] model('AcceptanceMetric',{'metric_id':enum(*metric_ids),'groups':arr(text(80),1,10),'unit':enum('ratio','seconds'),'direction':enum('ge','le'),'target':{'type':'number','minimum':0,'maximum':1000000},'min_denominator':integer(1),'min_scene_count':integer(),'min_positive_count':integer(),'min_negative_count':integer()}) @@ -28,7 +28,9 @@ def extend_release(g): metrics=[] def target(mid,value,direction='ge',minimum=100,scenes=30,groups=None,unit='ratio'): metrics.append({'metric_id':mid,'groups':groups or ['overall'],'unit':unit,'direction':direction,'target':value,'min_denominator':minimum,'min_scene_count':scenes,'min_positive_count':200 if mid.startswith('DET-') else (100 if mid.startswith('RISK-') else 0),'min_negative_count':100 if mid.startswith('RISK-') else 0}) - target('DET-P',.85,minimum=200,groups=['bottle','label','shelf','cabinet']);target('DET-R',.90,minimum=200,groups=['bottle','label','shelf','cabinet']) + # Per-class COCO metrics live in the evaluation report; the release contract + # keeps one bounded aggregate group to avoid an unbounded policy payload. + target('DET-P',.85,minimum=200,groups=['coco80']);target('DET-R',.90,minimum=200,groups=['coco80']) target('OCR-NAME',.90,minimum=200);target('OCR-DATE',.95) target('ENT-P',.98,minimum=200);target('ENT-COVER',.80,minimum=200);target('ENT-OOV',.99) target('Q-FALSE-REJECT',.10,'le');target('Q-FALSE-ACCEPT',.05,'le') diff --git a/tools/design/test_dfine_design.py b/tools/design/test_dfine_design.py index 54a5c7a..f00d466 100644 --- a/tools/design/test_dfine_design.py +++ b/tools/design/test_dfine_design.py @@ -1,7 +1,7 @@ """Synthetic D-FINE semantic tests; not trained-model, image-resize or GPU tests.""" import copy -from dfine_reference import (category_to_class, class_to_category, rgb_pixel_to_float, - query_outputs, adapt_outputs, check_request_limit) +from dfine_reference import (COCO_CATEGORY_IDS, category_to_class, class_to_category, + rgb_pixel_to_float, query_outputs, adapt_outputs, check_request_limit) def run_tests(): count = 0 @@ -17,25 +17,25 @@ def rejected(call): count += 1 else: raise AssertionError('invalid D-FINE input accepted') - for category in (1, 2, 3, 4): - ok(category_to_class(category) == category - 1) + for category in COCO_CATEGORY_IDS: ok(class_to_category(category_to_class(category)) == category) - for invalid in (0, 5, True, '1'): + for invalid in (0, 12, 91, True, '1'): rejected(lambda value=invalid: category_to_class(value)) - for invalid in (-1, 4, False, '0'): + for invalid in (-1, 80, False, '0'): rejected(lambda value=invalid: class_to_category(value)) ok(rgb_pixel_to_float([255, 0, 128]) == [1, 0, 128/255]) rejected(lambda: rgb_pixel_to_float([256, 0, 0])) - logits = [[-20.0]*4 for _ in range(300)] + logits = [[-20.0]*80 for _ in range(300)] boxes = [[.5, .5, .5, .5] for _ in range(300)] - logits[0] = [0, 5, 5, -2] - logits[1] = [0, 0, 0, 0] + logits[0][39] = 5 + logits[0][40] = 5 + logits[1] = [0]*80 labels, xyxy, scores = query_outputs(logits, boxes, [1600, 800]) - ok(labels[0] == 1 and labels[1] == 0) + ok(labels[0] == 39 and labels[1] == 0) ok(xyxy[0] == [400, 200, 1200, 600]) ok(scores[1] == .5 and len(scores) == 300) rows = adapt_outputs(labels, xyxy, scores, [1600, 800], .5) - ok(len(rows) == 2 and rows[0]['type'] == 'bottle' and rows[1]['type'] == 'label') + ok(len(rows) == 2 and rows[0]['type'] == 'person' and rows[1]['type'] == 'bottle') ok(rows[0]['bbox'] == [.25, .25, .75, .75]) ok(len(adapt_outputs(labels, xyxy, scores, [1600, 800], .500001)) == 1) ok(query_outputs(logits, boxes, [800, 1600])[1][0] == [200, 400, 600, 1200]) @@ -47,20 +47,21 @@ def rejected(call): bad = copy.deepcopy(logits); bad[0][2] = float('nan') rejected(lambda: query_outputs(bad, boxes, [1600, 800])) bad_box = copy.deepcopy(boxes); bad_box[0][2] = 0 - rejected(lambda: query_outputs(logits, bad_box, [1600, 800])) + bad_labels, bad_xyxy, bad_scores = query_outputs(logits, bad_box, [1600, 800]) + rejected(lambda: adapt_outputs(bad_labels, bad_xyxy, bad_scores, [1600, 800], .5)) rejected(lambda: query_outputs(logits, boxes, [0, 800])) bad_scores = scores.copy(); bad_scores[299] = float('inf') rejected(lambda: adapt_outputs(labels, xyxy, bad_scores, [1600, 800], .9)) rejected(lambda: adapt_outputs(labels, xyxy, scores, [1600, 800], -.1)) rejected(lambda: adapt_outputs(labels, xyxy[:-1], scores, [1600, 800], .9)) - bad_labels = labels.copy(); bad_labels[299] = 4 + bad_labels = labels.copy(); bad_labels[299] = 80 rejected(lambda: adapt_outputs(bad_labels, xyxy, scores, [1600, 800], .9)) ok(check_request_limit([[{}]*50, [{}]*50])) rejected(lambda: check_request_limit([[{}]*50, [{}]*51])) rejected(lambda: check_request_limit([])) # No NMS: distinct queries at identical coordinates are not silently merged. - duplicate_logits = [[-20.0]*4 for _ in range(300)] - duplicate_logits[0] = duplicate_logits[1] = [5, 0, 0, 0] + duplicate_logits = [[-20.0]*80 for _ in range(300)] + duplicate_logits[0] = duplicate_logits[1] = [5] + [0]*79 dl, db, ds = query_outputs(duplicate_logits, boxes, [1600, 800]) ok(len(adapt_outputs(dl, db, ds, [1600, 800], .9)) == 2) return count diff --git a/tools/design/test_release_design.py b/tools/design/test_release_design.py index effbae6..4b20270 100644 --- a/tools/design/test_release_design.py +++ b/tools/design/test_release_design.py @@ -10,7 +10,7 @@ def read(name):return json.loads((ROOT/'contracts'/name).read_text(encoding='utf def schema_check(name,value):Draft202012Validator(read(name),format_checker=FormatChecker()).validate(value) def fixture(): policy=read('development-acceptance-policy.json');policy.update(policy_id='SYNTHETIC-APPROVED-FOR-UNIT-TEST',status='approved') - manifest={'bundle_id':'22222222-2222-4222-8222-222222222222','version_label':'synthetic-unit-test','pipeline_version':'vision-v1','dictionary_version_id':'33333333-3333-4333-8333-333333333333','dictionary_sha256':'a'*64,'git_commit':'a'*40,'dataset_version':'SYNTHETIC-NO-REAL-DATA','device_profiles':['cuda'],'artifacts':[{'role':role,'object_key':'fixtures/'+role+'.json','sha256':digest({'synthetic':role}),'license':'SYNTHETIC TEST ONLY','runtime':'fixture'} for role in ['detector','ocr','quality','dictionary']],'thresholds':{'detection_min':.25,'ocr_min':.6,'entity_min':.9,'blur_min':80,'dark_min':.12,'glare_max':.3,'adjacent_gap_ratio':.25},'input_max_side':640,'detector_backend':'onnxruntime','ocr_backend':'paddleocr','evaluation_report_key':'fixtures/synthetic-evaluation.json','evaluation_passed':True,'purpose':'production','adapter_id':'dfine-n4-rgb-stretch-v1','model_family':'dfine_n','source_ref':'https://github.com/Peterande/D-FINE','source_commit':SOURCE_COMMIT,'runtime_profile':'dfine-cuda-fp32-ocrv4cpu-v1','calibration':{'status':'calibrated','validation_split_sha256':'b'*64,'report_sha256':'c'*64},'evaluation_policy_id':policy['policy_id'],'evaluation_policy_sha256':digest(policy)} + manifest={'bundle_id':'22222222-2222-4222-8222-222222222222','version_label':'synthetic-unit-test','pipeline_version':'vision-v1','dictionary_version_id':'33333333-3333-4333-8333-333333333333','dictionary_sha256':'a'*64,'git_commit':'a'*40,'dataset_version':'SYNTHETIC-NO-REAL-DATA','device_profiles':['cuda'],'artifacts':[{'role':role,'object_key':'fixtures/'+role+'.json','sha256':digest({'synthetic':role}),'license':'SYNTHETIC TEST ONLY','runtime':'fixture'} for role in ['detector','ocr','quality','dictionary']],'thresholds':{'detection_min':.25,'ocr_min':.6,'entity_min':.9,'blur_min':80,'dark_min':.12,'glare_max':.3,'adjacent_gap_ratio':.25},'input_max_side':640,'detector_backend':'onnxruntime','ocr_backend':'onnxruntime','evaluation_report_key':'fixtures/synthetic-evaluation.json','evaluation_passed':True,'purpose':'production','adapter_id':'dfine-coco80-rgb-stretch-v1','model_family':'dfine_n','source_ref':'https://github.com/Peterande/D-FINE','source_commit':SOURCE_COMMIT,'runtime_profile':'dfine-cuda-fp32-ocrv6smallcpu-v1','calibration':{'status':'calibrated','validation_split_sha256':'b'*64,'report_sha256':'c'*64},'evaluation_policy_id':policy['policy_id'],'evaluation_policy_sha256':digest(policy)} manifest['runtime']={'detector_device':'cuda','ocr_device':'cpu','detector_precision':'fp32','preprocess_id':'dfine-rgb-stretch640-v1','postprocess_id':'dfine-qmax-v1','onnx_opset':16,'runtime_lock_sha256':digest({'synthetic':'runtime-lock'})} manifest['content_sha256']=content_digest(manifest) measurements=[] @@ -42,12 +42,15 @@ def rejected(call): schema_check('model-manifest-v1.json',mock);count+=1 assert validate_manifest(mock,'test');count+=1 rejected(lambda:validate_manifest(mock,'production',TENANT,policy,report,registry)) - for field,value in [('content_sha256','0'*64),('adapter_id','unregistered'),('input_max_side',512),('runtime_profile','dfine-cpu-fp32-ocrv4cpu-v1'),('adapter_id','yolox-tiny4-decoded-v1'),('model_family','yolox_tiny'),('source_commit','b'*40)]: + for field,value in [('content_sha256','0'*64),('adapter_id','unregistered'),('input_max_side',512),('runtime_profile','dfine-cpu-fp32-ocrv6smallcpu-v1'),('adapter_id','yolox-tiny4-decoded-v1'),('model_family','yolox_tiny'),('source_commit','b'*40)]: bad=copy.deepcopy(manifest);bad[field]=value if field!='content_sha256':bad['content_sha256']=content_digest(bad) rejected(lambda:validate_manifest(bad,'dev')) - cpu=copy.deepcopy(dev);cpu['device_profiles']=['cpu'];cpu['runtime_profile']='dfine-cpu-fp32-ocrv4cpu-v1';cpu['runtime']['detector_device']='cpu';cpu['content_sha256']=content_digest(cpu) + cpu=copy.deepcopy(dev);cpu['device_profiles']=['cpu'];cpu['runtime_profile']='dfine-cpu-fp32-ocrv6smallcpu-v1';cpu['runtime']['detector_device']='cpu';cpu['content_sha256']=content_digest(cpu) schema_check('model-manifest-v1.json',cpu);assert validate_manifest(cpu,'dev');count+=2 + for field,value in [('ocr_backend','paddleocr'),('runtime_profile','dfine-cpu-fp32-ocrv4cpu-v1')]: + bad=copy.deepcopy(cpu);bad[field]=value;bad['content_sha256']=content_digest(bad) + rejected(lambda:validate_manifest(bad,'dev')) for key,value in [('detector_device','cpu'),('ocr_device','cuda'),('detector_precision','fp16'),('preprocess_id','letterbox'),('postprocess_id','nms'),('onnx_opset',17),('runtime_lock_sha256','not-a-hash')]: bad=copy.deepcopy(manifest);bad['runtime'][key]=value;bad['content_sha256']=content_digest(bad) rejected(lambda:validate_manifest(bad,'dev')) diff --git a/tools/design/validate_specs.py b/tools/design/validate_specs.py index aa8878c..8b2d92f 100644 --- a/tools/design/validate_specs.py +++ b/tools/design/validate_specs.py @@ -153,7 +153,7 @@ def main(): from test_readiness_design import run_tests as run_readiness_tests count('synthetic_readiness_checks',run_readiness_tests()) model_doc=(ROOT/'docs/04-ai-rules/02-model-data-plan.md').read_text(encoding='utf-8') - for token in [ADAPTER_ID,SOURCE_COMMIT,'dfine-cpu-fp32-ocrv4cpu-v1','dfine-cuda-fp32-ocrv4cpu-v1','dfine-qmax-v1','runtime_lock_sha256']: + for token in [ADAPTER_ID,SOURCE_COMMIT,'dfine-cpu-fp32-ocrv6smallcpu-v1','dfine-cuda-fp32-ocrv6smallcpu-v1','dfine-qmax-v1','runtime_lock_sha256']: check(token in model_doc,'model contract not documented: '+token);count('dfine_doc_tieouts') report={'status':'PASS','executed_at_utc':datetime.now(timezone.utc).isoformat(),'python_version':sys.version.split()[0],'counts':COUNT,'limitations':['No application runtime tests','No live MySQL DDL execution','No real model or expert safety-rule evaluation','No GPU, ONNX export or real-image resize validation','No live MinIO IAM, nginx or presigned URL validation','No human approval']} rendered=json.dumps(report,ensure_ascii=False,indent=2)+'\n' diff --git a/tools/ml/crop_checks.py b/tools/ml/crop_checks.py new file mode 100644 index 0000000..44bdba6 --- /dev/null +++ b/tools/ml/crop_checks.py @@ -0,0 +1,50 @@ +"""Parent-side reconstruction of real child-process OCR crop evidence.""" + +import hashlib + +from packages.image_evidence.perspective import rebuild_ocr_evidence + + +def verify_crops(report, paths): + import numpy as np + from PIL import Image + + checked, rotated, crop_ids = [], 0, set() + if len(report["images"]) != len(paths): + raise RuntimeError("Image count differs") + for row, path in zip(report["images"], paths): + if hashlib.sha256(path.read_bytes()).hexdigest() != row["input_sha256"]: + raise RuntimeError("Source file digest differs") + # Smoke inputs are normalized, metadata-free RGB PNGs, like analysis images. + with Image.open(path) as image: + rgb = np.asarray(image.convert("RGB"), dtype=np.uint8) + for line in row["lines"]: + metadata = line["crop_evidence"] + if ( + metadata["image_id"] != row["image_id"] + or metadata["line_id"] != line["line_id"] + or metadata["crop_id"] in crop_ids + or [[p["x"], p["y"]] for p in metadata["recipe"]["quad"]] != line["quad"] + ): + raise RuntimeError("Crop references or source quad differ") + crop_ids.add(metadata["crop_id"]) + evidence = rebuild_ocr_evidence(rgb, metadata) + rotated += metadata["recognition_rotation_ccw"] == 90 + checked.append( + { + "crop_id": metadata["crop_id"], + "png_sha256": evidence.sha256, + "png_size_bytes": len(evidence.png), + "recognition_rgb_sha256": metadata["recognition_rgb_sha256"], + "recognition_rotation_ccw": metadata["recognition_rotation_ccw"], + } + ) + return { + "status": "PASS", + "count": len(checked), + "rotated_count": rotated, + "source_hash_checked": True, + "png_hash_checked": True, + "recognition_pixels_checked": True, + "crops": checked, + } diff --git a/tools/ml/smoke_cpu.py b/tools/ml/smoke_cpu.py new file mode 100644 index 0000000..684e8c9 --- /dev/null +++ b/tools/ml/smoke_cpu.py @@ -0,0 +1,79 @@ +"""Repeatable synthetic-image smoke using a real, locally supplied official ONNX.""" + +import argparse +import json +import tempfile +import time +from datetime import datetime, timezone +from importlib.metadata import distributions +from pathlib import Path +from uuid import UUID + +from apps.ai_inference.cpu import CpuOptions, run_cpu_detection + + +def main(): + import numpy as np + from PIL import Image + + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + model = args.model.resolve() + options = CpuOptions( + str(model), + str(model.with_name("config.json")), + str(model.with_name("preprocessor_config.json")), + ) + run_id = str(UUID("11111111-1111-4111-8111-111111111111")) + with tempfile.TemporaryDirectory(prefix="labsafe-cpu-smoke-") as directory: + root = Path(directory) + paths = [root / name for name in ("blank.png", "portrait.png", "noise.png")] + with Image.new("RGB", (800, 400)) as image: + image.save(paths[0]) + pixels = np.zeros((800, 400, 3), dtype=np.uint8) + pixels[:, :200, 0] = 255 + pixels[:, 200:, 2] = 255 + with Image.fromarray(pixels) as image: + image.save(paths[1]) + pixels = np.random.default_rng(20261007).integers(0, 256, (400, 800, 3), dtype=np.uint8) + with Image.fromarray(pixels) as image: + image.save(paths[2]) + started = time.monotonic() + first = run_cpu_detection(options, paths, run_id) + wall_ms = round((time.monotonic() - started) * 1000) + second = run_cpu_detection(options, paths, run_id) + for left, right in zip(first["images"], second["images"]): + for key in ("image_id", "input_sha256", "width", "height", "diagnostics", "detections"): + if left[key] != right[key]: + raise RuntimeError("CPU smoke replay differs") + if left["diagnostics"]["candidates"] != 300: + raise RuntimeError("CPU smoke candidate count differs") + if first["runtime"]["providers"] != ["CPUExecutionProvider"]: + raise RuntimeError("Unexpected CPU provider") + evidence = { + "status": "PASS", + "executed_at_utc": datetime.now(timezone.utc).isoformat(), + "input_kind": "synthetic-images", + "bounded_process_wall_ms": wall_ms, + "replay_equal": True, + "installed_packages": dict( + sorted((d.metadata["Name"], d.version) for d in distributions()) + ), + "result": first, + "limitations": [ + "No real-photo accuracy evaluation", + "No OCR or business RPC", + "No CUDA evaluation", + "Not a production runtime lock or performance approval", + ], + } + args.output.write_text( + json.dumps(evidence, indent=2, ensure_ascii=False, allow_nan=False) + "\n", encoding="utf-8" + ) + print("CPU model smoke and replay: PASS") + + +if __name__ == "__main__": + main() diff --git a/tools/ml/smoke_ocr_cpu.py b/tools/ml/smoke_ocr_cpu.py new file mode 100644 index 0000000..c47f8ee --- /dev/null +++ b/tools/ml/smoke_ocr_cpu.py @@ -0,0 +1,83 @@ +"""Reproducible local real-weight OCR smoke with synthetic label images.""" + +import argparse +import hashlib +import json +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +from apps.ai_inference.ocr_cpu import OcrOptions, run_cpu_ocr +from tools.ml.crop_checks import verify_crops + + +def main(): + from PIL import Image, ImageDraw, ImageFont + + parser = argparse.ArgumentParser() + parser.add_argument("--det-dir", type=Path, required=True) + parser.add_argument("--rec-dir", type=Path, required=True) + parser.add_argument("--font", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + options = OcrOptions(str(args.det_dir.resolve()), str(args.rec_dir.resolve())) + font = ImageFont.truetype(str(args.font), 42) + expected = ["ETHANOL", "EXP 2027-12-31", "乙醇"] + with tempfile.TemporaryDirectory(prefix="labsafe-ocr-smoke-") as directory: + root = Path(directory) + paths = [root / name for name in ("blank.png", "label.png")] + with Image.new("RGB", (800, 400), "white") as image: + image.save(paths[0]) + draw = ImageDraw.Draw(image) + for index, text in enumerate(expected): + draw.text((60, 50 + 100 * index), text, font=font, fill="black") + image.save(paths[1]) + run_id = "11111111-1111-4111-8111-111111111111" + first = run_cpu_ocr(options, paths, run_id) + second = run_cpu_ocr(options, paths, run_id) + for left, right in zip(first["images"], second["images"]): + for key in ("image_id", "input_sha256", "width", "height", "lines"): + if left[key] != right[key]: + raise RuntimeError("OCR replay differs") + if first["images"][0]["lines"]: + raise RuntimeError("Blank image produced text regions") + actual = [row["text"] for row in first["images"][1]["lines"]] + if actual != expected: + raise RuntimeError(f"Synthetic label differs: {actual!r}") + if any(row["uncertain"] for row in first["images"][1]["lines"]): + raise RuntimeError("Synthetic label confidence below the development threshold") + crops = verify_crops(first, paths) + evidence = { + "status": "PASS", + "executed_at_utc": datetime.now(timezone.utc).isoformat(), + "input_kind": "synthetic-images", + "replay_equal": True, + "crop_reconstruction": crops, + "font_sha256": hashlib.sha256(args.font.read_bytes()).hexdigest(), + "font_size": 42, + "synthetic_ground_truth": expected, + "result": first, + "provided_files": { + role: { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in directory.iterdir() + if path.is_file() + } + for role, directory in (("det", args.det_dir), ("rec", args.rec_dir)) + }, + "limitations": [ + "No real-photo accuracy evaluation", + "No chemical entity or date facts", + "No bottle association or HTTP business RPC", + "No orientation classifier", + "No Paddle reference comparison or production approval", + ], + } + args.output.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + print("Real CPU OCR synthetic-label smoke and replay: PASS") + + +if __name__ == "__main__": + main() diff --git a/tools/ml/smoke_pipeline_cpu.py b/tools/ml/smoke_pipeline_cpu.py new file mode 100644 index 0000000..d9585fa --- /dev/null +++ b/tools/ml/smoke_pipeline_cpu.py @@ -0,0 +1,121 @@ +"""Real joint CPU model smoke with strict quality gating and reproducible inputs.""" + +import argparse +import hashlib +import json +import tempfile +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path + +from apps.ai_inference.pipeline_cpu import PipelineOptions, run_cpu_pipeline +from tools.ml.crop_checks import verify_crops + + +def main(): + from PIL import Image, ImageDraw, ImageFont + + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--det-dir", type=Path, required=True) + parser.add_argument("--rec-dir", type=Path, required=True) + parser.add_argument("--font", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + options = PipelineOptions( + str(args.model.resolve()), str(args.det_dir.resolve()), str(args.rec_dir.resolve()) + ) + font = ImageFont.truetype(str(args.font), 42) + expected = ["ETHANOL", "EXP 2027-12-31", "乙醇"] + run_id = "11111111-1111-4111-8111-111111111111" + with tempfile.TemporaryDirectory(prefix="labsafe-pipeline-smoke-") as directory: + root = Path(directory) + paths = [root / name for name in ("landscape.png", "portrait.png")] + for path, size in zip(paths, ((800, 400), (400, 800))): + with Image.new("RGB", size, (180, 180, 180)) as image: + draw = ImageDraw.Draw(image) + for index, text in enumerate(expected): + draw.text((40, 50 + 100 * index), text, font=font, fill="black") + image.save(path) + rotated = root / "rotated-clockwise90.png" + with Image.open(paths[0]) as image, image.transpose(Image.Transpose.ROTATE_270) as turned: + turned.save(rotated) + paths.append(rotated) + blank = root / "black.png" + with Image.new("RGB", (800, 400)) as image: + image.save(blank) + first = run_cpu_pipeline(options, paths, run_id) + second = run_cpu_pipeline(options, paths, run_id) + if first["outcome"] != "needs_review" or not first["models_executed"]: + raise RuntimeError("Clean synthetic images did not pass the real quality gate") + for index, (left, right) in enumerate(zip(first["images"], second["images"])): + for key in ( + "image_id", + "input_sha256", + "quality", + "detections", + "lines", + "detection_diagnostics", + "inference_executed", + ): + if left[key] != right[key]: + raise RuntimeError("Joint CPU replay differs") + actual = [row["text"] for row in left["lines"]] + if actual != expected if index < 2 else sorted(actual) != sorted(expected): + raise RuntimeError("Synthetic OCR label differs") + if any(row["uncertain"] for row in left["lines"]): + raise RuntimeError("Synthetic OCR confidence below development threshold") + if left["detection_diagnostics"]["candidates"] != 300: + raise RuntimeError("Detector candidate count differs") + crops = verify_crops(first, paths) + if crops["count"] != 9 or crops["rotated_count"] != 3: + raise RuntimeError("Expected nine crops including three rotated recognition regions") + mixed = run_cpu_pipeline(options, [paths[0], blank], run_id) + if mixed["outcome"] != "needs_retake" or mixed["models_executed"] or mixed["artifacts"]: + raise RuntimeError("Failed-image whole-batch gate did not stop model loading") + if any( + row["inference_executed"] or row["detections"] or row["lines"] + for row in mixed["images"] + ): + raise RuntimeError("Mixed batch leaked partial detections or OCR") + quality = run_cpu_pipeline(replace(options, quality_only=True), paths, run_id) + if quality["outcome"] != "quality_passed" or quality["models_executed"]: + raise RuntimeError("Quality-only branch ran model inference") + if [row["quality"] for row in quality["images"]] != [ + row["quality"] for row in first["images"] + ]: + raise RuntimeError("Quality-only/joint quality scores differ") + evidence = { + "status": "PASS", + "executed_at_utc": datetime.now(timezone.utc).isoformat(), + "input_kind": "synthetic-images", + "replay_equal": True, + "quality_stage_equal": True, + "crop_reconstruction": crops, + "font_sha256": hashlib.sha256(args.font.read_bytes()).hexdigest(), + "generation": { + "font_size": 42, + "background_rgb": [180, 180, 180], + "origins": [[40, 50], [40, 150], [40, 250]], + "ground_truth": expected, + "image_sizes": [[800, 400], [400, 800], [400, 800]], + "third_image_rotation": "clockwise90-from-first-image", + }, + "joint_result": first, + "mixed_retake_result": mixed, + "quality_only_result": quality, + "limitations": [ + "Synthetic engineering smoke, not real-photo accuracy", + "No chemical/date facts or bottle association", + "No HTTP/Worker RPC", + "No production runtime lock, performance approval or CUDA evaluation", + ], + } + args.output.write_text( + json.dumps(evidence, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + print("Real CPU quality + COCO80 + OCR joint smoke, batch gate and replay: PASS") + + +if __name__ == "__main__": + main()