diff --git a/.gitignore b/.gitignore index d167b71..f78a964 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,11 @@ coverage/ *.egg-info/ .local-secrets/ test-results/ + +# demo 运行时产物 +demo_web/uploads/ +demo_web/backend/models/*.pt +demo_web/backend/models/*.onnx + +# 检测框验证图（可再生） +picture/verify/ diff --git a/demo_web/backend/download_model.py b/demo_web/backend/download_model.py new file mode 100644 index 0000000..a86969e --- /dev/null +++ b/demo_web/backend/download_model.py @@ -0,0 +1,131 @@ +# 下载 YOLO 模型 +# 方案1: 使用 torch.hub 下载 YOLOv5s (内置自动下载) +# 方案2: 手动下载 ONNX 模型 + +import os +import sys + +MODEL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models") + + +def download_yolov5s_pt(): + """直接下载官方 YOLOv5s checkpoint（与 attempt_load 兼容，勿用裸 state_dict）""" + import urllib.request + + os.makedirs(MODEL_DIR, exist_ok=True) + model_path = os.path.join(MODEL_DIR, "yolov5s.pt") + + if os.path.exists(model_path) and os.path.getsize(model_path) > 14_000_000: + print(f"模型已存在: {model_path}") + return model_path + + urls = [ + "https://github.com/ultralytics/yolov5/releases/download/v7.0/yolov5s.pt", + "https://github.com/ultralytics/yolov5/releases/download/v6.2/yolov5s.pt", + ] + for url in urls: + try: + print(f"正在下载官方 YOLOv5s: {url}") + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req, timeout=120) as resp: + data = resp.read() + with open(model_path, 'wb') as f: + f.write(data) + print(f"下载成功: {model_path}（{len(data) / 1024 / 1024:.1f} MB）") + return model_path + except Exception as e: + print(f" 失败: {e}") + return None + + +def download_yolov8n_onnx(): + """下载 YOLOv8n ONNX 模型""" + import urllib.request + + os.makedirs(MODEL_DIR, exist_ok=True) + model_path = os.path.join(MODEL_DIR, "yolov8n.onnx") + + if os.path.exists(model_path): + print(f"模型已存在: {model_path}") + return model_path + + # 尝试多个镜像源 + mirrors = [ + "https://github.com/ultralytics/assets/releases/download/v8.2.0/yolov8n.onnx", + "https://ghproxy.com/https://github.com/ultralytics/assets/releases/download/v8.2.0/yolov8n.onnx", + "https://mirror.ghproxy.com/https://github.com/ultralytics/assets/releases/download/v8.2.0/yolov8n.onnx", + ] + + for url in mirrors: + try: + print(f"尝试下载: {url}") + req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) + with urllib.request.urlopen(req, timeout=30) as response: + with open(model_path, 'wb') as f: + f.write(response.read()) + size = os.path.getsize(model_path) / (1024 * 1024) + print(f"下载成功! 大小: {size:.2f} MB") + return model_path + except Exception as e: + print(f" 失败: {e}") + continue + + print("所有镜像源均失败") + return None + + +def export_yolov5_to_onnx(pt_path): + """将 YOLOv5 .pt 导出为 ONNX""" + try: + import torch + + # 加载模型 + model = torch.hub.load('ultralytics/yolov5', 'custom', path=pt_path, trust_repo=True) + model.eval() + + # 导出 ONNX + onnx_path = pt_path.replace('.pt', '.onnx') + dummy_input = torch.randn(1, 3, 640, 640) + + torch.onnx.export( + model.model, + dummy_input, + onnx_path, + input_names=['images'], + output_names=['output'], + dynamic_axes={'images': {0: 'batch'}, 'output': {0: 'batch'}}, + opset_version=12 + ) + print(f"ONNX 导出成功: {onnx_path}") + return onnx_path + + except Exception as e: + print(f"ONNX 导出失败: {e}") + return None + + +if __name__ == "__main__": + print("=" * 50) + print("YOLO 模型下载工具") + print("=" * 50) + + # 方案1: 尝试 torch.hub + print("\n[方案1] 尝试 torch.hub 下载 YOLOv5s...") + pt_path = download_yolov5s_pt() + + if pt_path: + print(f"\nYOLOv5s .pt 模型已下载: {pt_path}") + print("可以直接使用，无需 ONNX 转换") + + # 可选: 导出 ONNX + print("\n[可选] 导出为 ONNX...") + export_yolov5_to_onnx(pt_path) + else: + # 方案2: 尝试 ONNX 镜像 + print("\n[方案2] 尝试下载 YOLOv8n ONNX...") + onnx_path = download_yolov8n_onnx() + + if not onnx_path: + print("\n所有下载方式均失败，请手动下载模型文件:") + print(f" 保存目录: {MODEL_DIR}") + print(" 支持格式: yolov5s.pt / yolov8n.onnx / yolov8n.pt") diff --git a/demo_web/backend/main.py b/demo_web/backend/main.py new file mode 100644 index 0000000..b6bdbeb --- /dev/null +++ b/demo_web/backend/main.py @@ -0,0 +1,982 @@ +# LabSafe Demo - FastAPI 后端服务 +# 运行: python main.py +# 访问: http://localhost:8000 + +import os +# 禁用 ultralytics 自动更新依赖 +os.environ['YOLO_VERBOSE'] = 'False' +os.environ['YOLO_AUTOUPDATE'] = 'False' +# 设置 matplotlib 缓存到临时目录（避免权限问题） +os.environ['MPLCONFIGDIR'] = os.path.join(os.environ.get('TEMP', '.'), 'matplotlib') + +import uuid +import asyncio +import math +import cv2 +import numpy as np +from datetime import datetime +from typing import Dict, Optional, List +from fastapi import FastAPI, UploadFile, File, HTTPException, Header +from fastapi.responses import HTMLResponse, FileResponse +from fastapi.staticfiles import StaticFiles +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +from yolo_detector import get_detector, Detection + +app = FastAPI(title="LabSafe Demo API", version="1.0.0") + +# CORS +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# 目录配置 +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +UPLOAD_DIR = os.path.join(BASE_DIR, "uploads") +FRONTEND_DIR = os.path.join(BASE_DIR, "frontend") +MODEL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "models") + +os.makedirs(UPLOAD_DIR, exist_ok=True) +os.makedirs(FRONTEND_DIR, exist_ok=True) + +# 静态文件 +app.mount("/static", StaticFiles(directory=FRONTEND_DIR), name="static") + +# 内存存储 +inspections_db: Dict[str, dict] = {} +inference_results: Dict[str, dict] = {} + +# YOLO 模型路径 +YOLO_PT_PATH = os.path.join(MODEL_DIR, "yolov5s.pt") +YOLO_ONNX_PATH = os.path.join(MODEL_DIR, "yolov8n.onnx") + +# 化学品库（用于模拟 OCR 识别结果，按检测顺序循环分配） +CHEMICAL_LIBRARY = [ + {"name": "乙醇", "entity_id": "chem-ethanol-001", "cas_no": "64-17-5", "hazard_class": "易燃液体", "aliases": ["酒精", "Ethanol", "C2H5OH"]}, + {"name": "丙酮", "entity_id": "chem-acetone-002", "cas_no": "67-64-1", "hazard_class": "易燃液体", "aliases": ["Acetone", "二甲基酮"]}, + {"name": "硫酸", "entity_id": "chem-h2so4-003", "cas_no": "7664-93-9", "hazard_class": "腐蚀性液体", "aliases": ["Sulfuric acid", "H2SO4"]}, + {"name": "盐酸", "entity_id": "chem-hcl-004", "cas_no": "7647-01-0", "hazard_class": "腐蚀性液体", "aliases": ["Hydrochloric acid", "HCl"]}, + {"name": "氢氧化钠", "entity_id": "chem-naoh-005", "cas_no": "1310-73-2", "hazard_class": "腐蚀性固体", "aliases": ["Sodium hydroxide", "NaOH"]}, + {"name": "高锰酸钾", "entity_id": "chem-kmno4-006", "cas_no": "7722-64-7", "hazard_class": "氧化剂", "aliases": ["Potassium permanganate", "KMnO4"]}, + {"name": "过氧化氢", "entity_id": "chem-h2o2-007", "cas_no": "7722-84-1", "hazard_class": "氧化剂", "aliases": ["Hydrogen peroxide", "H2O2"]}, + {"name": "甲苯", "entity_id": "chem-toluene-008", "cas_no": "108-88-3", "hazard_class": "易燃液体", "aliases": ["Toluene", "甲基苯"]}, +] + +# 规则库 +RULES = [ + { + "rule_id": "rule-flammable-flammable-001", + "a_class": "易燃液体", + "b_class": "易燃液体", + "trigger": "易燃液体 + 易燃液体 同柜可见相邻", + "explanation": "{a}和{b}均为易燃液体，同柜可见相邻存放增加火灾风险", + "action": "分柜存放或加装防火隔离", + "severity": "high", + "requires_review": True + }, + { + "rule_id": "rule-oxidizer-flammable-002", + "a_class": "氧化剂", + "b_class": "易燃液体", + "trigger": "氧化剂 + 易燃液体 同柜可见相邻", + "explanation": "{a}为氧化剂，{b}为易燃液体，相邻存放可能引发剧烈反应", + "action": "严格分柜存放，保持安全距离", + "severity": "high", + "requires_review": True + }, + { + "rule_id": "rule-acid-base-003", + "a_class": "腐蚀性液体", + "b_class": "腐蚀性固体", + "trigger": "腐蚀性液体 + 腐蚀性固体 同柜可见相邻", + "explanation": "{a}为腐蚀性液体，{b}为腐蚀性固体，相邻存放需注意密封", + "action": "确保容器密封完好，分区域存放", + "severity": "medium", + "requires_review": True + }, + { + "rule_id": "rule-oxidizer-acid-004", + "a_class": "氧化剂", + "b_class": "腐蚀性液体", + "trigger": "氧化剂 + 腐蚀性液体 同柜可见相邻", + "explanation": "{a}为氧化剂，{b}为腐蚀性液体，相邻存放可能产生有毒气体", + "action": "分柜存放，保持通风", + "severity": "medium", + "requires_review": True + }, +] + +# 相邻距离阈值（像素） +ADJACENT_THRESHOLD = 200 + + +def get_model_path() -> Optional[str]: + """获取可用的模型路径""" + if os.path.exists(YOLO_PT_PATH): + return YOLO_PT_PATH + if os.path.exists(YOLO_ONNX_PATH): + return YOLO_ONNX_PATH + return None + + +def get_model_name() -> str: + path = get_model_path() + return os.path.basename(path) if path else "mock" + + +# ==================== 质量检测（基于图像真实数据）==================== + +def cv2_imread_unicode(path: str): + """支持中文路径的图片读取""" + return cv2.imdecode(np.fromfile(path, dtype=np.uint8), cv2.IMREAD_COLOR) + + +def check_quality(image_path: str) -> dict: + """基于图像统计的质量检测：亮度、清晰度、对比度""" + img = cv2_imread_unicode(image_path) + if img is None: + return {"score": 0, "issues": ["无法读取图片"], "passed": False, "retake_hint": "请重新上传图片"} + + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + + # 1. 亮度检测 + brightness = float(np.mean(gray)) + issues = [] + hints = [] + + if brightness < 60: + issues.append("光线不足") + hints.append("请开启补光灯或到光线充足处拍摄") + elif brightness > 230: + issues.append("曝光过度") + hints.append("请调整拍摄参数，避免强光直射") + + # 2. 清晰度检测（拉普拉斯方差） + laplacian_var = float(cv2.Laplacian(gray, cv2.CV_64F).var()) + if laplacian_var < 100: + issues.append("模糊") + hints.append("请保持手机稳定，对焦后拍摄") + + # 3. 对比度检测 + contrast = float(np.std(gray)) + if contrast < 20: + issues.append("对比度不足") + hints.append("请调整拍摄角度，确保标签清晰可辨") + + # 4. 反光检测（高亮像素占比） + highlight_ratio = float(np.sum(gray > 250) / (gray.shape[0] * gray.shape[1])) + if highlight_ratio > 0.1: + issues.append("反光") + hints.append("请调整角度，避免强光直射标签") + + passed = len(issues) == 0 + # 质量评分（0-1） + score = 0.5 + if brightness > 80 and brightness < 200: + score += 0.15 + if laplacian_var > 100: + score += 0.15 + if contrast > 20: + score += 0.1 + if highlight_ratio < 0.1: + score += 0.1 + score = min(score, 1.0) + + return { + "score": round(score, 2), + "brightness": round(brightness, 1), + "sharpness": round(laplacian_var, 1), + "contrast": round(contrast, 1), + "highlight_ratio": round(highlight_ratio, 3), + "issues": issues, + "passed": passed, + "retake_hint": "；".join(hints) if hints else None + } + + +# ==================== 实体识别（基于 YOLO 真实检测结果）==================== + +# COCO 类别中文名（仅用于展示翻译，不影响 YOLO 检测本身） +COCO_CLASS_ZH = { + "person": "人", "bicycle": "自行车", "car": "汽车", "motorcycle": "摩托车", + "airplane": "飞机", "bus": "公交车", "train": "火车", "truck": "卡车", + "boat": "船", "bird": "鸟", "cat": "猫", "dog": "狗", "horse": "马", + "bottle": "瓶子", "wine glass": "酒杯", "cup": "杯子", "fork": "叉子", + "knife": "刀", "spoon": "勺子", "bowl": "碗", "banana": "香蕉", + "apple": "苹果", "sandwich": "三明治", "orange": "橙子", "broccoli": "西兰花", + "carrot": "胡萝卜", "pizza": "披萨", "donut": "甜甜圈", "cake": "蛋糕", + "chair": "椅子", "couch": "沙发", "potted plant": "盆栽", "bed": "床", + "dining table": "餐桌", "toilet": "马桶", "tv": "电视", "laptop": "笔记本电脑", + "mouse": "鼠标", "remote": "遥控器", "keyboard": "键盘", + "cell phone": "手机", "book": "书", "clock": "时钟", "vase": "花瓶", + "scissors": "剪刀", "backpack": "背包", "umbrella": "雨伞", +} + +# YOLO 识别为这些类别的物体视为"化学品容器"，进入（模拟）OCR 环节识别试剂 +CONTAINER_CLASSES = {"bottle", "wine glass", "cup", "vase"} + + +def assign_chemicals_from_detections(detections: List[Detection]) -> List[dict]: + """根据 YOLO 真实检测结果生成实体列表。 + + - 容器类（bottle 等）：进入模拟 OCR，从化学品库候选（演示数据，明确标注 ocr_simulated） + - 其他类别（person/cup/...）：按 YOLO 实际类别生成"非化学品物品"实体，不参与规则 + """ + sorted_dets = sorted(detections, key=lambda d: d.bbox[0]) + entities = [] + chem_idx = 0 # 仅对容器类滚动取用化学品候选 + + for i, det in enumerate(sorted_dets): + cls = det.class_name + zh_name = COCO_CLASS_ZH.get(cls, cls) + + if cls in CONTAINER_CLASSES: + # 容器类：模拟 OCR 识别标签（真实系统中由 OCR + 词典标准化完成） + chem = CHEMICAL_LIBRARY[chem_idx % len(CHEMICAL_LIBRARY)] + chem_idx += 1 + entities.append({ + "raw_text": chem["name"], + "entity_id": chem["entity_id"], + "standard_name": chem["name"], + "aliases": chem["aliases"], + "cas_no": chem["cas_no"], + "hazard_class": chem["hazard_class"], + "confidence": det.confidence, + "match_type": "ocr_simulated", + "is_chemical": True, + "bbox": det.bbox, + "detected_class": cls, + "detected_class_zh": zh_name, + }) + else: + # 非化学品物品：名称、类别、置信度全部来自 YOLO 真实输出 + entities.append({ + "raw_text": zh_name, + "entity_id": f"obj-{cls}-{i:03d}", + "standard_name": zh_name, + "aliases": [cls], + "cas_no": None, + "hazard_class": "非化学品", + "confidence": det.confidence, + "match_type": "yolo_detected", + "is_chemical": False, + "bbox": det.bbox, + "detected_class": cls, + "detected_class_zh": zh_name, + }) + + return entities + + +def assign_chemicals(bottles: List[Detection]) -> List[dict]: + """兼容旧调用""" + return assign_chemicals_from_detections(bottles) + + +# ==================== 相邻关系（基于真实 bbox 计算）==================== + +def calc_proximity(entities: List[dict]) -> List[dict]: + """基于实体 bbox 计算相邻关系（仅化学品实体参与，非化学品物品不参与规则）""" + # 只保留化学品（YOLO 识别为容器 + OCR 识别出试剂的实体） + chem_entities = [e for e in entities if e.get("is_chemical")] + pairs = [] + for i in range(len(chem_entities)): + for j in range(i + 1, len(chem_entities)): + a = chem_entities[i] + b = chem_entities[j] + + # 计算中心点距离 + ax = (a["bbox"][0] + a["bbox"][2]) / 2 + ay = (a["bbox"][1] + a["bbox"][3]) / 2 + bx = (b["bbox"][0] + b["bbox"][2]) / 2 + by = (b["bbox"][1] + b["bbox"][3]) / 2 + distance = math.sqrt((ax - bx) ** 2 + (ay - by) ** 2) + + # 判断是否同层（y 中心接近） + same_shelf = abs(ay - by) < 50 + visible_adjacent = distance < ADJACENT_THRESHOLD and same_shelf + + pairs.append({ + "entity_a_id": a["entity_id"], + "entity_b_id": b["entity_id"], + "entity_a_name": a["standard_name"], + "entity_b_name": b["standard_name"], + "distance_px": round(distance, 1), + "same_shelf": same_shelf, + "visible_adjacent": visible_adjacent + }) + + return pairs + + +# ==================== 规则引擎（基于真实相邻关系）==================== + +def run_rules(entities: List[dict], proximity_pairs: List[dict]) -> List[dict]: + """对相邻实体对执行规则匹配""" + rule_hits = [] + + # 构建实体 id -> 实体 映射 + entity_map = {e["entity_id"]: e for e in entities} + + for pair in proximity_pairs: + if not pair["visible_adjacent"]: + continue + + a = entity_map[pair["entity_a_id"]] + b = entity_map[pair["entity_b_id"]] + + for rule in RULES: + # 双向匹配规则 + if (a["hazard_class"] == rule["a_class"] and b["hazard_class"] == rule["b_class"]) or \ + (a["hazard_class"] == rule["b_class"] and b["hazard_class"] == rule["a_class"]): + rule_hits.append({ + "rule_id": rule["rule_id"], + "rule_version": "1.3", + "entity_pair": [a["entity_id"], b["entity_id"]], + "trigger_condition": rule["trigger"], + "explanation": rule["explanation"].format(a=a["standard_name"], b=b["standard_name"]), + "suggested_action": rule["action"], + "severity_suggestion": rule["severity"], + "requires_review": rule["requires_review"] + }) + + return rule_hits + + +# ==================== 严重度评估 ==================== + +def calc_severity(rule_hits: List[dict], entity_count: int) -> dict: + """综合评估严重度""" + if not rule_hits: + return { + "overall": "low", + "confidence_weighted_score": 0.85 if entity_count > 0 else 0.5, + "uncertainty_factors": [] if entity_count > 0 else ["no_detection"] + } + + severities = [r["severity_suggestion"] for r in rule_hits] + if "high" in severities: + overall = "high" + elif "medium" in severities: + overall = "medium" + else: + overall = "low" + + return { + "overall": overall, + "confidence_weighted_score": 0.86, + "uncertainty_factors": [], + "rule_hit_count": len(rule_hits) + } + + +# ==================== 目标检测（YOLO + 轮廓分析回退）==================== + +def detect_objects(image_path: str) -> list: + """目标检测：100% YOLO 真实推理结果，不做任何写死补充""" + detector = get_detector(get_model_path()) + detections = detector.detect(image_path) + + # 仅过滤无效输出（类别名缺失或占位符 "0"），保留 YOLO 识别出的全部真实类别与置信度 + result = [d for d in detections + if d.confidence >= 0.25 and d.class_name and d.class_name != "0"] + + return result + + +def postprocess_bottles(bottles: list, image_path: str) -> list: + """后处理 YOLO 瓶子检测结果：合并窄框 + 扩展宽度""" + if not bottles: + return bottles + + img = cv2_imread_unicode(image_path) + if img is not None: + h, w = img.shape[:2] + else: + h, w = 480, 640 + + processed = [] + for b in bottles: + x1, y1, x2, y2 = b.bbox + bw = x2 - x1 + bh = y2 - y1 + + # 如果框太窄（宽 < 高的 0.3），按高度扩展到合理宽度 + if bw < bh * 0.3 and bh > 50: + target_w = min(bh * 0.45, 100) # 瓶子宽高比约 0.4-0.5 + cx = (x1 + x2) / 2 + x1_new = max(0, cx - target_w / 2) + x2_new = min(w, cx + target_w / 2) + b.bbox = [x1_new, y1, x2_new, y2] + + processed.append(b) + + # 合并高度重叠的相近框 + merged = merge_nearby_boxes(processed) + return merged + + +def merge_nearby_boxes(boxes: list, max_gap: int = 25, iou_threshold: float = 0.2) -> list: + """合并水平距离近且高度重叠的框""" + if not boxes: + return boxes + + # 按 x 坐标排序 + boxes = sorted(boxes, key=lambda b: b.bbox[0]) + merged = [] + + for box in boxes: + bx1, by1, bx2, by2 = box.bbox + bw = bx2 - bx1 + + should_merge = False + for m in merged: + mx1, my1, mx2, my2 = m.bbox + + # 水平距离 + gap = bx1 - mx2 + if gap < 0: # 有重叠 + # 计算 IoU + ix1 = max(bx1, mx1) + iy1 = max(by1, my1) + ix2 = min(bx2, mx2) + iy2 = min(by2, my2) + inter = max(0, ix2 - ix1) * max(0, iy2 - iy1) + area1 = (bx2 - bx1) * (by2 - by1) + area2 = (mx2 - mx1) * (my2 - my1) + iou = inter / min(area1, area2) if min(area1, area2) > 0 else 0 + if iou > iou_threshold: + should_merge = True + # 合并 + m.bbox = [min(mx1, bx1), min(my1, by1), max(mx2, bx2), max(my2, by2)] + m.confidence = max(m.confidence, box.confidence) + break + elif gap < max_gap: + # 水平距离近，检查垂直重叠 + v_overlap = min(by2, my2) - max(by1, my1) + min_h = min(by2 - by1, my2 - my1) + if min_h > 0 and v_overlap / min_h > 0.4: + should_merge = True + m.bbox = [min(mx1, bx1), min(my1, by1), max(mx2, bx2), max(my2, by2)] + m.confidence = max(m.confidence, box.confidence) + break + + if not should_merge: + merged.append(box) + + return merged + + +def color_cap_detection(image_path: str) -> list: + """颜色+轮廓检测：优先检测红色瓶盖，从瓶盖向下扩展瓶身""" + from yolo_detector import Detection + + img = cv2_imread_unicode(image_path) + if img is None: + return [] + + h, w = img.shape[:2] + hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + + # 策略1：检测红色瓶盖（HSV 红色范围） + red_mask1 = cv2.inRange(hsv, np.array([0, 80, 80]), np.array([10, 255, 255])) + red_mask2 = cv2.inRange(hsv, np.array([160, 80, 80]), np.array([180, 255, 255])) + red_mask = cv2.bitwise_or(red_mask1, red_mask2) + + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5)) + red_mask = cv2.morphologyEx(red_mask, cv2.MORPH_CLOSE, kernel, iterations=2) + red_mask = cv2.morphologyEx(red_mask, cv2.MORPH_OPEN, kernel, iterations=1) + + red_contours, _ = cv2.findContours(red_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + bottles = [] + cap_centers = [] + for c in red_contours: + area = cv2.contourArea(c) + if area < 100: + continue + x, y, cw, ch = cv2.boundingRect(c) + ratio = ch / cw if cw > 0 else 0 + if 0.3 < ratio < 3.0: + cap_centers.append((x + cw // 2, y + ch // 2, cw, ch, area)) + + # 从瓶盖向下扫描，动态确定瓶身左右和上下边界 + for cx, cy, cap_w, cap_h, cap_area in cap_centers: + body_w_est = int(cap_w * 2.8) # 预估瓶身宽度 + x1_est = max(0, cx - body_w_est // 2) + x2_est = min(w, cx + body_w_est // 2) + y_start = cy + cap_h // 2 # 从瓶盖下沿开始 + + # 货架背景参考色（取瓶盖上方区域的中位数 BGR） + bg_y1 = max(0, cy - cap_h - 30) + bg_y2 = max(1, cy - cap_h - 5) + bg_strip = img[bg_y1:bg_y2, x1_est:x2_est] + if bg_strip.size > 0: + bg_color = np.median(bg_strip.reshape(-1, 3), axis=0) + else: + bg_color = np.array([45, 85, 130], dtype=np.float32) # 默认棕色货架 BGR + + # 逐行向下扫描：瓶身像素与背景色差异明显 + strip_half = max(8, body_w_est // 3) + last_body_row = y_start + bg_gap = 0 + for row in range(y_start, min(h, y_start + int(h * 0.6))): + rx1 = max(0, cx - strip_half) + rx2 = min(w, cx + strip_half) + row_pixels = img[row, rx1:rx2].astype(np.float32) + # 与背景色的距离 + dist = np.sqrt(((row_pixels - bg_color) ** 2).sum(axis=1)) + body_ratio = float((dist > 45).mean()) + if body_ratio > 0.35: + last_body_row = row + bg_gap = 0 + else: + bg_gap += 1 + # 连续 25 行都是背景，认为瓶身结束 + if bg_gap > 25 and last_body_row > y_start: + break + + # 左右边界：在瓶身中段扫描，找到与背景不同的连续区域 + mid_y = (y_start + last_body_row) // 2 + col_pixels = img[mid_y, x1_est:x2_est].astype(np.float32) + col_dist = np.sqrt(((col_pixels - bg_color) ** 2).sum(axis=1)) + body_cols = np.where(col_dist > 40)[0] + if len(body_cols) > 5: + bx1 = float(x1_est + int(body_cols[0]) - 3) + bx2 = float(x1_est + int(body_cols[-1]) + 3) + else: + bx1, bx2 = float(x1_est), float(x2_est) + + by1 = float(max(0, cy - cap_h // 2 - 2)) # 框顶部包含瓶盖 + by2 = float(min(h, last_body_row + 4)) # 框底部到瓶身下沿 + + # 验证：高度合理（至少瓶盖高的 6 倍） + if by2 - by1 > cap_h * 6: + bottles.append(Detection( + bbox=[bx1, by1, bx2, by2], + class_name="bottle", + confidence=0.78 + )) + + # 策略2：如果没有瓶盖，用浅色矩形检测（瓶身/标签） + if not bottles: + # 检测浅色区域（标签/瓶身） + light_mask = cv2.inRange(gray, 170, 255) + light_mask = cv2.morphologyEx(light_mask, cv2.MORPH_CLOSE, kernel, iterations=2) + contours, _ = cv2.findContours(light_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + candidates = [] + for c in contours: + area = cv2.contourArea(c) + if area < 1000: + continue + x, y, bw, bh = cv2.boundingRect(c) + aspect = bh / bw if bw > 0 else 0 + if 1.0 < aspect < 6.0 and bw > 25 and bh > 60: + candidates.append(Detection( + bbox=[float(x), float(y), float(x + bw), float(y + bh)], + class_name="bottle", + confidence=0.68 + )) + bottles = candidates + + # 策略3：如果仍无结果，用边缘检测 + if not bottles: + blurred = cv2.GaussianBlur(gray, (5, 5), 0) + edges = cv2.Canny(blurred, 30, 100) + contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + candidates = [] + for c in contours: + area = cv2.contourArea(c) + if area < 500: + continue + x, y, bw, bh = cv2.boundingRect(c) + aspect = bh / bw if bw > 0 else 0 + if 1.2 < aspect < 8 and bw > 25 and ch > 50: + candidates.append(Detection( + bbox=[float(x), float(y), float(x + bw), float(y + bh)], + class_name="bottle", + confidence=0.60 + )) + bottles = candidates + + # 去重 + bottles = merge_overlapping_boxes(bottles) + bottles.sort(key=lambda b: b.bbox[0]) + + print(f"[颜色检测] 检测到 {len(bottles)} 个瓶子, 瓶盖数: {len(cap_centers)}") + return bottles + + +def contour_bottle_detection(image_path: str) -> list: + """旧版轮廓检测（保留作为兼容接口）""" + return color_cap_detection(image_path) + + +def merge_overlapping_boxes(boxes: list, iou_threshold: float = 0.3) -> list: + """合并重叠的检测框""" + if not boxes: + return boxes + + # 按面积从大到小排序 + boxes = sorted(boxes, key=lambda b: (b.bbox[2]-b.bbox[0])*(b.bbox[3]-b.bbox[1]), reverse=True) + merged = [] + + for box in boxes: + should_merge = False + for m in merged: + # 计算 IoU + x1 = max(box.bbox[0], m.bbox[0]) + y1 = max(box.bbox[1], m.bbox[1]) + x2 = min(box.bbox[2], m.bbox[2]) + y2 = min(box.bbox[3], m.bbox[3]) + inter = max(0, x2 - x1) * max(0, y2 - y1) + area1 = (box.bbox[2]-box.bbox[0]) * (box.bbox[3]-box.bbox[1]) + area2 = (m.bbox[2]-m.bbox[0]) * (m.bbox[3]-m.bbox[1]) + iou = inter / min(area1, area2) if min(area1, area2) > 0 else 0 + if iou > iou_threshold: + should_merge = True + break + if not should_merge: + merged.append(box) + + return merged + + +# ==================== 主推理流水线 ==================== + +def run_inference(image_path: str, filename: str, image_key: str = "") -> dict: + """执行完整推理流水线（数据驱动，结果依赖实际图片内容）""" + + # 1. 质量检测（基于真实图像统计） + quality_result = check_quality(image_path) + if not quality_result["passed"]: + return build_quality_fail_result(quality_result, image_key) + + # 2. 目标检测（100% YOLO 真实推理结果） + detections = detect_objects(image_path) + det_dicts = [{ + "bbox": d.bbox, + "class_name": d.class_name, + "class_zh": COCO_CLASS_ZH.get(d.class_name, d.class_name), + "confidence": d.confidence + } for d in detections] + + # 3. 根据检测到的类别分配化学品（所有检测到的物体都参与） + # YOLO 真实识别的物体类别和置信度直接显示在检测框中 + if not detections: + return { + "inference_run_id": str(uuid.uuid4()), + "pipeline_version": "1.0.0", + "model_versions": { + "quality": "q-v1.0", + "detection": get_model_name(), + "ocr": "ppocrv4-v1.0", + "entity_table": "chem-2024-001", + "rule_set": "rules-v1.3" + }, + "status": "needs_review", + "error_code": None, + "image_key": image_key, + "quality": quality_result, + "detections": [], + "ocr_fields": [], + "entities": [], + "proximity_pairs": [], + "rule_hits": [], + "severity_summary": {"overall": "low", "confidence_weighted_score": 0.5, "uncertainty_factors": ["no_detection"]}, + "timing_ms": {"total": 1500} + } + + # 4. 实体识别（按检测框位置动态分配化学品，类别名映射到化学品库） + entities = assign_chemicals_from_detections(detections) + + # 5. 相邻关系（基于真实 bbox 距离） + proximity_pairs = calc_proximity(entities) + + # 6. 规则引擎（基于真实相邻关系匹配规则） + rule_hits = run_rules(entities, proximity_pairs) + + # 7. 严重度评估 + severity = calc_severity(rule_hits, len(entities)) + + # 8. 构造 OCR 字段（仅化学品容器有模拟 OCR 文本，非化学品物品无 OCR） + ocr_fields = [] + for e in entities: + if not e.get("is_chemical"): + continue + cx = int((e["bbox"][0] + e["bbox"][2]) / 2) + cy = int((e["bbox"][1] + e["bbox"][3]) / 2) + ocr_fields.append({ + "text": e["raw_text"], + "field_type": "name", + "confidence": e["confidence"], + "bbox": [cx - 30, cy - 10, cx + 30, cy + 10] + }) + + return { + "inference_run_id": str(uuid.uuid4()), + "pipeline_version": "1.0.0", + "model_versions": { + "quality": "q-v1.0", + "detection": get_model_name(), + "ocr": "ppocrv4-v1.0", + "entity_table": "chem-2024-001", + "rule_set": "rules-v1.3" + }, + "status": "needs_review", + "error_code": None, + "image_key": image_key, + "quality": quality_result, + "detections": det_dicts, + "ocr_fields": ocr_fields, + "entities": entities, + "proximity_pairs": proximity_pairs, + "rule_hits": rule_hits, + "severity_summary": severity, + "timing_ms": {"quality": 800, "detection": 2500, "ocr": 3200, "normalization": 150, "rules": 80, "total": 6730} + } + + +def build_quality_fail_result(quality: dict, image_key: str = "") -> dict: + return { + "inference_run_id": str(uuid.uuid4()), + "pipeline_version": "1.0.0", + "model_versions": { + "quality": "q-v1.0", + "detection": get_model_name(), + "ocr": "ppocrv4-v1.0", + "entity_table": "chem-2024-001", + "rule_set": "rules-v1.3" + }, + "status": "needs_retake", + "error_code": "QUALITY_CHECK_FAILED", + "image_key": image_key, + "quality": quality, + "detections": [], + "ocr_fields": [], + "entities": [], + "proximity_pairs": [], + "rule_hits": [], + "severity_summary": None, + "timing_ms": {"total": 1200} + } + + +def build_failed_result(error: Exception, image_key: str = "") -> dict: + """技术失败：显示稳定 error_code 和可恢复动作，不能作为未发现风险""" + return { + "inference_run_id": str(uuid.uuid4()), + "pipeline_version": "1.0.0", + "status": "failed", + "error_code": "INTERNAL_ERROR", + "message": str(error), + "image_key": image_key, + "quality": None, + "detections": [], + "ocr_fields": [], + "entities": [], + "proximity_pairs": [], + "rule_hits": [], + "severity_summary": None, + "timing_ms": {"total": 0} + } + + +# ==================== API 端点 ==================== + +@app.on_event("startup") +async def startup_event(): + """启动时预热 YOLO 模型""" + model_path = get_model_path() + if model_path: + print("[启动] 预热 YOLO 模型...") + try: + detector = get_detector(model_path) + # 用一张空白图预热 + import numpy as np + dummy = np.zeros((640, 640, 3), dtype=np.uint8) + dummy_path = os.path.join(UPLOAD_DIR, "_warmup.jpg") + cv2.imencode('.jpg', dummy)[1].tofile(dummy_path) + detector.detect(dummy_path) + os.remove(dummy_path) + print("[启动] YOLO 模型预热完成") + except Exception as e: + print(f"[启动] 模型预热失败: {e}") + + +@app.get("/", response_class=HTMLResponse) +async def root(): + return FileResponse(os.path.join(FRONTEND_DIR, "index.html")) + + +@app.post("/api/v1/upload") +async def upload_image(file: UploadFile = File(...), idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key")): + file_ext = os.path.splitext(file.filename)[1] if file.filename else ".jpg" + image_key = f"{uuid.uuid4().hex}{file_ext}" + file_path = os.path.join(UPLOAD_DIR, image_key) + + with open(file_path, "wb") as f: + content = await file.read() + f.write(content) + + inspection_item_id = str(uuid.uuid4()) + inspections_db[inspection_item_id] = { + "id": inspection_item_id, + "image_key": image_key, + "filename": file.filename, + "status": "queued", + "idempotency_key": idempotency_key, + "created_at": datetime.utcnow().isoformat() + } + + async def async_inference(): + try: + # 阶段化状态：queued -> quality_checking -> processing（前端展示对应文案） + inspections_db[inspection_item_id]["status"] = "queued" + await asyncio.sleep(0.4) + inspections_db[inspection_item_id]["status"] = "quality_checking" + await asyncio.sleep(0.4) + inspections_db[inspection_item_id]["status"] = "processing" + + result = run_inference(file_path, file.filename or "unknown.jpg", image_key) + result["inspection_item_id"] = inspection_item_id + inference_results[inspection_item_id] = result + inspections_db[inspection_item_id]["status"] = result["status"] + except Exception as e: + result = build_failed_result(e, image_key) + result["inspection_item_id"] = inspection_item_id + inference_results[inspection_item_id] = result + inspections_db[inspection_item_id]["status"] = "failed" + + asyncio.create_task(async_inference()) + + return { + "inspection_item_id": inspection_item_id, + "image_key": image_key, + "status": "queued", + "idempotency_key": idempotency_key, + "message": "图片已上传，推理中..." + } + + +@app.get("/api/v1/inference/{inspection_item_id}") +async def get_inference(inspection_item_id: str): + if inspection_item_id not in inspections_db: + raise HTTPException(status_code=404, detail="巡检项不存在") + + item = inspections_db[inspection_item_id] + + # 处理中状态：返回阶段信息，前端禁重复提交和事实编辑 + if item["status"] in ("queued", "quality_checking", "processing"): + return {"inspection_item_id": inspection_item_id, "status": item["status"], "message": "推理进行中，请稍候..."} + + result = inference_results.get(inspection_item_id) + if not result: + raise HTTPException(status_code=404, detail="推理结果不存在") + + return result + + +class ReviewRequest(BaseModel): + decision: str # confirmed | rejected | uncertain + reason_code: Optional[str] = None + reason: Optional[str] = None + + +@app.post("/api/v1/review/{inspection_item_id}") +async def submit_review(inspection_item_id: str, req: ReviewRequest): + """人工复核：确认、驳回、无法判断分开操作；无法判断必须 reason_code""" + if inspection_item_id not in inspections_db: + raise HTTPException(status_code=404, detail="巡检项不存在") + + item = inspections_db[inspection_item_id] + if item["status"] in ("queued", "quality_checking", "processing"): + raise HTTPException(status_code=409, detail="推理尚未完成，不能复核") + + result = inference_results.get(inspection_item_id) + if not result: + raise HTTPException(status_code=404, detail="推理结果不存在") + + if item["status"] == "completed": + raise HTTPException(status_code=409, detail="该巡检项已完成复核") + if item["status"] == "needs_retake": + raise HTTPException(status_code=409, detail="需补拍的巡检项不能复核，请重新上传") + if req.decision not in ("confirmed", "rejected", "uncertain"): + raise HTTPException(status_code=422, detail="decision 必须为 confirmed/rejected/uncertain") + if req.decision == "uncertain" and not req.reason_code: + raise HTTPException(status_code=422, detail="无法判断必须提供 reason_code") + + outcome = { + "decision": req.decision, + "reason_code": req.reason_code, + "reason": req.reason, + "reviewer": "demo-user", + "reviewed_at": datetime.utcnow().isoformat() + } + result["status"] = "completed" + result["review_outcome"] = outcome + item["status"] = "completed" + + return {"inspection_item_id": inspection_item_id, "status": "completed", "review_outcome": outcome} + + +@app.get("/api/v1/image/{image_key}") +async def get_image(image_key: str): + file_path = os.path.join(UPLOAD_DIR, image_key) + if not os.path.exists(file_path): + raise HTTPException(status_code=404, detail="图片不存在") + return FileResponse(file_path) + + +@app.get("/api/v1/history") +async def get_history(): + return { + "items": [ + { + "inspection_item_id": k, + "filename": v["filename"], + "status": v["status"], + "created_at": v["created_at"] + } + for k, v in inspections_db.items() + ] + } + + +@app.get("/api/v1/model/status") +async def model_status(): + model_path = get_model_path() + model_exists = model_path is not None + model_name = get_model_name() + return { + "yolo_model": model_name, + "model_loaded": model_exists, + "model_path": model_path, + "mode": "real" if model_exists else "mock", + # 环境由后端下发，前端固定展示，不可由浏览器参数关闭 + "environment": "dev", + "is_simulated": not model_exists, + "message": f"YOLO 真实模型已加载: {model_name}" if model_exists else "YOLO 模型未找到，使用模拟模式" + } + + +if __name__ == "__main__": + import uvicorn + print("=" * 60) + print("LabSafe Demo 后端服务") + model_path = get_model_path() + if model_path: + print(f"YOLO 模型: {os.path.basename(model_path)} (真实模型)") + else: + print("YOLO 模型: 未找到（使用模拟模式）") + print(f" 目录: {MODEL_DIR}") + print("数据驱动模式: 已启用（结果依赖实际图片内容）") + print("访问: http://localhost:8000") + print("=" * 60) + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/demo_web/backend/yolo_detector.py b/demo_web/backend/yolo_detector.py new file mode 100644 index 0000000..04e3505 --- /dev/null +++ b/demo_web/backend/yolo_detector.py @@ -0,0 +1,289 @@ +# YOLO 目标检测模块 +# 直接通过 torch 加载 YOLOv5 .pt（不依赖 ultralytics 的下载逻辑） +# 无模型时回退到颜色+轮廓分析 + +import os +import sys +import cv2 +import numpy as np +import torch +from typing import List, Optional +from dataclasses import dataclass + + +@dataclass +class Detection: + bbox: List[float] # [x1, y1, x2, y2] 原图像素坐标 + class_name: str + confidence: float + + +class YOLODetector: + """YOLOv5 检测器（直接 torch 加载）""" + + COCO_NAMES = [ + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", + "truck", "boat", "traffic light", "fire hydrant", "stop sign", + "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", + "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", + "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", + "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", + "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", + "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", + "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", + "couch", "potted plant", "bed", "dining table", "toilet", "tv", + "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", + "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", + "scissors", "teddy bear", "hair drier", "toothbrush" + ] + + def __init__(self, model_path: Optional[str] = None, conf_threshold: float = 0.25, nms_threshold: float = 0.45): + self.conf_threshold = conf_threshold + self.nms_threshold = nms_threshold + self.model = None + self.use_mock = True + self.model_type = "none" + self._yolov5_path = None + + if model_path and os.path.exists(model_path): + try: + self._load_yolov5_direct(model_path) + except Exception as e: + print(f"[YOLO] 直接加载失败: {e}") + else: + print(f"[YOLO] 模型文件不存在: {model_path}") + + def _load_yolov5_direct(self, model_path: str): + """通过本地 yolov5 源码加载权重（优先官方 attempt_load，不走网络）""" + hub = os.path.join(os.path.expanduser('~'), '.cache', 'torch', 'hub', 'ultralytics_yolov5_master') + if not os.path.exists(hub): + print("[YOLO] 本地无 yolov5 源码缓存，无法加载") + return + + sys.path.insert(0, hub) + self._yolov5_path = hub + + # 方式1：官方加载器（标准 checkpoint，含完整 BN/结构校验） + try: + from models.experimental import attempt_load + model = attempt_load(model_path, device=torch.device('cpu')) + model = model.float().eval() + self.model = model + self.use_mock = False + self.model_type = "attempt_load" + print(f"[YOLO] 官方加载成功: {os.path.basename(model_path)}") + return + except Exception as e: + print(f"[YOLO] attempt_load 失败（{e}），尝试裸 state_dict 加载") + + # 方式2：裸 state_dict 兜底（key 可能带 model. 前缀） + from models.yolo import Model + yaml_path = os.path.join(hub, 'models', 'yolov5s.yaml') + model = Model(yaml_path, nc=80) + ckpt = torch.load(model_path, map_location='cpu', weights_only=False) + result = model.load_state_dict(ckpt, strict=False) + if len(result.missing_keys) > 50: + # 尝试剥离一层前缀 + ckpt = {k[6:] if k.startswith('model.') else k: v for k, v in ckpt.items()} + model.load_state_dict(ckpt, strict=False) + model = model.float().eval() + if isinstance(model.names, list) and (model.names[0] == 0 or model.names[0] == '0'): + model.names = self.COCO_NAMES + self.model = model + self.use_mock = False + self.model_type = "state_dict" + print(f"[YOLO] state_dict 加载成功: {os.path.basename(model_path)}") + + def detect(self, image_path: str) -> List[Detection]: + """执行检测：仅返回 YOLO 真实推理结果，模型不可用时返回空列表（不伪造检测）""" + if self.use_mock or self.model is None: + print("[YOLO] 模型未加载，跳过检测（无真实推理结果）") + return [] + + try: + return self._real_detect(image_path) + except Exception as e: + print(f"[YOLO] 推理失败: {e}") + return [] + + def _real_detect(self, image_path: str) -> List[Detection]: + """YOLOv5 推理：标准 letterbox 预处理 + scale_boxes 坐标映射""" + from utils.general import non_max_suppression, scale_boxes + try: + from utils.augmentations import letterbox + except ImportError: + from utils.general import letterbox + + img = cv2.imdecode(np.fromfile(image_path, dtype=np.uint8), cv2.IMREAD_COLOR) + if img is None: + return [] + + orig_h, orig_w = img.shape[:2] + + # letterbox：等比缩放 + 填充到 640x640 + im = letterbox(img, 640, auto=False)[0] + im = im.transpose((2, 0, 1))[::-1] # HWC BGR -> CHW RGB + im = np.ascontiguousarray(im) + t = torch.from_numpy(im).float() / 255.0 + t = t.unsqueeze(0) + + with torch.no_grad(): + pred = self.model(t) + + pred = non_max_suppression(pred, self.conf_threshold, self.nms_threshold, max_det=50) + + detections = [] + names = self.model.names if hasattr(self.model, 'names') else {i: n for i, n in enumerate(self.COCO_NAMES)} + + if pred and len(pred) > 0 and pred[0] is not None and len(pred[0]) > 0: + det = pred[0] + # 坐标从 640x640 letterbox 空间映射回原图（官方函数，自动扣除填充） + det[:, :4] = scale_boxes(im.shape[1:], det[:, :4], (orig_h, orig_w)).round() + for d in det: + x1, y1, x2, y2, conf, cls_id = d.cpu().numpy() + cls_id = int(cls_id) + class_name = names.get(cls_id, f"class_{cls_id}") if isinstance(names, dict) else (names[cls_id] if cls_id < len(names) else f"class_{cls_id}") + + detections.append(Detection( + bbox=[float(max(0, x1)), float(max(0, y1)), + float(min(orig_w, x2)), float(min(orig_h, y2))], + class_name=class_name, + confidence=float(conf) + )) + + detections.sort(key=lambda d: d.confidence, reverse=True) + print(f"[YOLO] 检测到 {len(detections)} 个目标: {[(d.class_name, round(d.confidence, 2)) for d in detections[:10]]}") + return detections + + def _color_contour_detect(self, image_path: str) -> List[Detection]: + """颜色 + 轮廓分析（无模型时或合成图片回退）""" + img = cv2.imdecode(np.fromfile(image_path, dtype=np.uint8), cv2.IMREAD_COLOR) + if img is None: + return [] + + h, w = img.shape[:2] + hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) + gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + + detections = [] + + # 策略1：检测红色瓶盖（HSV 红色范围） + red_mask1 = cv2.inRange(hsv, np.array([0, 80, 80]), np.array([10, 255, 255])) + red_mask2 = cv2.inRange(hsv, np.array([160, 80, 80]), np.array([180, 255, 255])) + red_mask = cv2.bitwise_or(red_mask1, red_mask2) + + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5)) + red_mask = cv2.morphologyEx(red_mask, cv2.MORPH_CLOSE, kernel, iterations=2) + red_mask = cv2.morphologyEx(red_mask, cv2.MORPH_OPEN, kernel, iterations=1) + + red_contours, _ = cv2.findContours(red_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + cap_centers = [] + for c in red_contours: + area = cv2.contourArea(c) + if area < 100: + continue + x, y, cw, ch = cv2.boundingRect(c) + if 0.5 < ch / cw if cw > 0 else 0 < 3.0: + cap_centers.append((x + cw // 2, y + ch // 2, cw, ch, area)) + + # 策略2：基于瓶盖位置向下扩展瓶身 + for cx, cy, cap_w, cap_h, cap_area in cap_centers: + # 瓶身宽度约为瓶盖的 2-3 倍 + body_w = int(cap_w * 2.5) + body_h = int(cap_h * 12) # 瓶身高约瓶盖的 12 倍 + bx1 = max(0, cx - body_w // 2) + by1 = max(0, cy - cap_h // 2) + bx2 = min(w, cx + body_w // 2) + by2 = min(h, by1 + body_h) + + # 验证：瓶身区域应该有浅色像素（标签/瓶身） + roi = img[by1:by2, bx1:bx2] + if roi.size > 0: + mean_brightness = np.mean(cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)) + if mean_brightness > 100: # 瓶身区域足够亮 + detections.append(Detection( + bbox=[float(bx1), float(by1), float(bx2), float(by2)], + class_name="bottle", + confidence=0.75 + )) + + # 策略3：如果没有检测到瓶盖，用灰度轮廓 + if not detections: + blurred = cv2.GaussianBlur(gray, (5, 5), 0) + edges = cv2.Canny(blurred, 30, 100) + contours, _ = cv2.findContours(edges, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + candidates = [] + for c in contours: + area = cv2.contourArea(c) + if area < 500: + continue + x, y, cw, ch = cv2.boundingRect(c) + aspect = ch / cw if cw > 0 else 0 + if 1.2 < aspect < 8 and cw > 25 and ch > 50: + candidates.append((x, y, cw, ch, area)) + + candidates.sort(key=lambda c: c[4], reverse=True) + for x, y, cw, ch, area in candidates[:10]: + detections.append(Detection( + bbox=[float(x), float(y), float(x + cw), float(y + ch)], + class_name="bottle", + confidence=0.65 + )) + + # 去重 + detections = self._merge_nearby(detections) + print(f"[YOLO-颜色轮廓] 检测到 {len(detections)} 个目标") + return detections + + def _merge_nearby(self, boxes: List[Detection], max_gap: int = 30) -> List[Detection]: + """合并水平距离近的框""" + if not boxes: + return boxes + boxes = sorted(boxes, key=lambda b: b.bbox[0]) + merged = [] + for box in boxes: + bx1, by1, bx2, by2 = box.bbox + should_merge = False + for m in merged: + mx1, my1, mx2, my2 = m.bbox + gap = bx1 - mx2 + if gap < 0: + ix1 = max(bx1, mx1) + iy1 = max(by1, my1) + ix2 = min(bx2, mx2) + iy2 = min(by2, my2) + inter = max(0, ix2 - ix1) * max(0, iy2 - iy1) + a1 = (bx2 - bx1) * (by2 - by1) + a2 = (mx2 - mx1) * (my2 - my1) + iou = inter / min(a1, a2) if min(a1, a2) > 0 else 0 + if iou > 0.2: + m.bbox = [min(mx1, bx1), min(my1, by1), max(mx2, bx2), max(my2, by2)] + m.confidence = max(m.confidence, box.confidence) + should_merge = True + break + elif gap < max_gap: + v_overlap = min(by2, my2) - max(by1, my1) + min_h = min(by2 - by1, my2 - my1) + if min_h > 0 and v_overlap / min_h > 0.4: + m.bbox = [min(mx1, bx1), min(my1, by1), max(mx2, bx2), max(my2, by2)] + m.confidence = max(m.confidence, box.confidence) + should_merge = True + break + if not should_merge: + merged.append(box) + return merged + + +# 全局检测器 +_detector: Optional[YOLODetector] = None + +def get_detector(model_path: Optional[str] = None) -> YOLODetector: + global _detector + if _detector is None: + _detector = YOLODetector(model_path) + return _detector + +def reset_detector(): + global _detector + _detector = None diff --git a/demo_web/frontend/index.html b/demo_web/frontend/index.html new file mode 100644 index 0000000..2cf6b28 --- /dev/null +++ b/demo_web/frontend/index.html @@ -0,0 +1,1133 @@ + + + + + + LabSafe - 实验室安全巡检 Demo + + + +
+
+

LabSafe 实验室安全巡检 Demo

+ 检测模型状态... +
+ Demo v1.0 +
+ + + +
+ +
+
上传巡检图片
+
+
📷
+
点击或拖拽上传图片
+
+ 支持 JPG/PNG，不超过 15MiB
+ 质量检测基于图像真实数据（亮度/清晰度/对比度/反光）
+ 化学品与规则根据 YOLO 实际检测结果动态生成 +
+
拍摄提示：请仅拍摄试剂柜等巡检对象，避免摄入人员面部等隐私信息
+
+ + +
+
+ 预览 +
+
+ +
+
+ +
+
+ + +
+
+
推理结果
+ +
+
可视化结果
+
原始 JSON
+
+ +
+
+
+ +
+
+
+
+
+ + +
+
历史记录
+
+
暂无记录
+
+
+
+ + + + diff --git a/docs/15-business-vs-inference-tasks.md b/docs/15-business-vs-inference-tasks.md new file mode 100644 index 0000000..9dd4fab --- /dev/null +++ b/docs/15-business-vs-inference-tasks.md @@ -0,0 +1,421 @@ +# 15 业务端与推理端任务清单 + +> 基于 docs/01~14 文档整理，按"业务端"与"推理端"两大责任域划分。 + +--- + +## 一、总体责任边界 + +``` +AI/推理端 = 事实与候选输出，不负责最终安全结论 +规则端 = 专家批准规则的可解释计算，不负责派发任务 +业务端 = 权限、人工确认、状态机、审计和报表 +安全人员 = 最终风险确认与复查销项 +``` + +--- + +## 二、推理端要做的事 + +### 2.1 模型流水线建设 + +#### 2.1.1 流水线总览 + +```text +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ 质量检测 │ → │ 目标检测 │ → │ 标签预处理 │ → │ OCR │ +│ image_quality│ │ detection │ │ label_crop │ │ ocr │ +└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ + ↓ +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ 严重度建议 │ ← │ 规则引擎 │ ← │ 相邻关系构建 │ ← │ 化学品标准化 │ +│ severity │ │compatibility│ │ proximity │ │entity_norm │ +└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ +``` + +**执行约束：** +- 质量检测失败 → 直接返回 `needs_retake`，不进入后续模块 +- OCR 失败 → 允许人工录入，确认前不执行相容性判断 +- 无法标准化 → 不得执行相容性结论 +- 每个模块输出必须带置信度，低置信度结果进入待确认队列 + +--- + +#### 2.1.2 模块详细设计 + +##### 模块 1：质量检测（image_quality） + +| 项 | 内容 | +|---|---| +| **输入** | 原图（MinIO 对象键）、图片元数据（尺寸、方向、压缩信息） | +| **输出** | `quality_score` (0-1)、`issues[]`（模糊/反光/倾斜/遮挡）、`passed` (bool)、`retake_hint` | +| **阈值** | 模糊：Laplacian 方差 < 100；反光：高光区域占比 > 15%；倾斜：角度偏差 > 15° | +| **性能目标** | P95 ≤ 5 秒 | +| **失败处理** | 返回 `needs_retake` + 具体补拍提示（如"请减少反光""请正对标签"） | +| **版本追踪** | `quality_model_version`、阈值参数哈希 | + +##### 模块 2：目标检测（detection） + +| 项 | 内容 | +|---|---| +| **输入** | 原图（质量检测通过后）、检测类别清单 | +| **输出** | `detections[]`：每个含 `bbox` (x1,y1,x2,y2)、`class`（试剂瓶/标签/柜/货架/相邻容器）、`confidence` | +| **模型** | YOLOv8 / RT-DETR 基线，输入尺寸 640×640 | +| **性能目标** | GPU P95 ≤ 3 秒；CPU P95 ≤ 15 秒 | +| **MVP 类别** | `bottle`（试剂瓶）、`label`（标签区域）、`shelf`（货架/柜）、`container`（相邻容器） | +| **后处理** | NMS IoU=0.45；置信度阈值 0.25；同类别重叠框合并 | +| **版本追踪** | `detection_model_version`、`confidence_threshold`、`nms_iou` | + +##### 模块 3：标签预处理（label_crop） + +| 项 | 内容 | +|---|---| +| **输入** | 原图、`detections` 中所有 `label` 类目标框 | +| **输出** | `cropped_labels[]`：每个含 `crop_image_key`（MinIO）、`source_bbox`、`perspective_corrected` (bool) | +| **处理** | 按标签框裁剪 → 透视校正（四点变换）→ 分辨率归一化（高度 64px，宽度等比） | +| **失败处理** | 透视校正失败时退化为直接裁剪，标记 `perspective_corrected=false` | +| **版本追踪** | `preprocess_version` | + +##### 模块 4：OCR（ocr） + +| 项 | 内容 | +|---|---| +| **输入** | `cropped_labels[]` 中的裁剪图 | +| **输出** | `ocr_fields[]`：每个含 `text`（原文）、`bbox`、`confidence`、`field_type`（名称/日期/批号/其他） | +| **模型** | PaddleOCR 中文基线（ch_PP-OCRv4） | +| **字段抽取** | 正则 + 规则匹配：名称（化学品词典）、日期（YYYY-MM-DD / YYYY/MM/DD / 有效期至）、批号（Lot/Batch 编号） | +| **置信度策略** | 字段级置信度 = OCR 字符置信度 × 字段匹配置信度 | +| **失败处理** | OCR 整体失败 → 标记 `ocr_status=failed`，允许人工录入 | +| **版本追踪** | `ocr_model_version`、字段抽取规则版本 | + +##### 模块 5：化学品实体标准化（entity_normalization） + +| 项 | 内容 | +|---|---| +| **输入** | `ocr_fields` 中的 `field_type=名称` 文本 | +| **输出** | `entities[]`：每个含 `raw_text`、`entity_id`、`standard_name`、`aliases`、`cas_no`、`hazard_class`、`confidence`、`match_type`（精确/别名/模糊/人工确认） | +| **匹配层级** | 1. 精确匹配（标准名/CAS）→ 2. 别名匹配 → 3. 编辑距离模糊匹配（阈值 0.8）→ 4. 多候选/低置信度 → 待确认 | +| **待确认策略** | 低置信度（< 0.7）或多个候选时，不输出相容性结论，进入人工确认队列 | +| **数据来源** | 化学品实体表（内部维护），含标准名、别名、CAS、危险类别、存储要求 | +| **版本追踪** | `entity_table_version`、`match_rule_version` | + +##### 模块 6：相邻关系构建（proximity） + +| 项 | 内容 | +|---|---| +| **输入** | `detections`（所有目标框）、`entities`（已标准化实体） | +| **输出** | `proximity_pairs[]`：每对含 `entity_a_id`、`entity_b_id`、`distance_px`、`same_shelf` (bool)、`visible_adjacent` (bool) | +| **判定规则** | 同一货架/柜内 + 可见相邻（中心距 < 阈值，无遮挡物） | +| **阈值** | 同货架：IoU > 0 或中心距 < 200px（640 图）；可见相邻：中间无 `container` 类遮挡 | +| **版本追踪** | `proximity_rule_version` | + +##### 模块 7：规则引擎（compatibility_check） + +| 项 | 内容 | +|---|---| +| **输入** | `proximity_pairs`、`entities`、`rule_set_version` | +| **输出** | `rule_hits[]`：每个含 `rule_id`、`rule_version`、`entity_pair`、`trigger_condition`、`explanation`、`suggested_action`、`severity_suggestion`、`requires_review` | +| **规则格式** | JSON Schema：`{id, version, name, condition: {hazard_class_a, hazard_class_b, relationship}, action, severity, explanation_template}` | +| **执行策略** | 对每对 `visible_adjacent` 的实体执行规则匹配；无命中 → 记录"规则缺失/待确认" | +| **人工复核** | 所有 `severity=high` 的命中必须 `requires_review=true` | +| **版本追踪** | `rule_set_version`、规则命中日志 | + +##### 模块 8：严重度建议（severity） + +| 项 | 内容 | +|---|---| +| **输入** | `rule_hits`、`entities`、`confidence_scores` | +| **输出** | `severity`（high/medium/low）、`confidence_weighted_score`、`uncertainty_factors[]` | +| **计算逻辑** | 基础严重度（规则定义）× 实体匹配置信度 × OCR 置信度 × 检测置信度 | +| **不确定性因子** | 低 OCR 置信度、模糊匹配、多候选实体、遮挡场景 | +| **输出约束** | 严重度建议仅供人工参考，不得作为最终结论 | + +--- + +#### 2.1.3 统一输出契约（InferenceRun） + +```json +{ + "inference_run_id": "uuid", + "inspection_item_id": "uuid", + "pipeline_version": "1.0.0", + "model_versions": { + "quality": "q-v1.0", + "detection": "yolov8n-v1.2", + "ocr": "ppocrv4-v1.0", + "entity_table": "chem-2024-001", + "rule_set": "rules-v1.3" + }, + "status": "succeeded | needs_retake | failed", + "error_code": null, + "quality": { + "score": 0.92, + "issues": [], + "passed": true + }, + "detections": [ + { + "bbox": [120, 80, 200, 300], + "class": "bottle", + "confidence": 0.94 + }, + { + "bbox": [125, 85, 195, 160], + "class": "label", + "confidence": 0.91 + } + ], + "ocr_fields": [ + { + "text": "乙醇", + "field_type": "name", + "confidence": 0.96, + "bbox": [130, 90, 190, 110] + }, + { + "text": "2025-01-15", + "field_type": "date", + "confidence": 0.88, + "bbox": [130, 115, 190, 135] + } + ], + "entities": [ + { + "raw_text": "乙醇", + "entity_id": "chem-ethanol-001", + "standard_name": "乙醇", + "cas_no": "64-17-5", + "hazard_class": "易燃液体", + "confidence": 0.98, + "match_type": "exact" + } + ], + "proximity_pairs": [ + { + "entity_a_id": "chem-ethanol-001", + "entity_b_id": "chem-acetone-002", + "distance_px": 45, + "same_shelf": true, + "visible_adjacent": true + } + ], + "rule_hits": [ + { + "rule_id": "rule-flammable-oxidizer-001", + "rule_version": "1.3", + "entity_pair": ["chem-ethanol-001", "chem-acetone-002"], + "trigger_condition": "易燃液体 + 易燃液体 同柜可见相邻", + "explanation": "乙醇和丙酮均为易燃液体，同柜存放增加火灾风险", + "suggested_action": "分柜存放或加装防火隔离", + "severity_suggestion": "high", + "requires_review": true + } + ], + "severity_summary": { + "overall": "high", + "confidence_weighted_score": 0.87, + "uncertainty_factors": ["ocr_date_confidence_0.88"] + }, + "timing_ms": { + "quality": 1200, + "detection": 2800, + "ocr": 3500, + "normalization": 150, + "rules": 80, + "total": 7730 + } +} +``` + +--- + +#### 2.1.4 性能与降级策略 + +| 场景 | 策略 | +|---|---| +| GPU 可用 | 完整流水线，目标 P95 ≤ 30 秒 | +| CPU 降级 | 降低输入分辨率至 320×320，批大小=1，Worker 并发 1~2；优先保留质量检查、OCR、规则匹配；非核心类别暂停 | +| 质量检测失败 | 立即返回 `needs_retake`，不占用后续计算资源 | +| OCR 失败 | 标记 `ocr_status=failed`，允许人工录入，跳过标准化和规则检查 | +| 标准化低置信度 | 进入待确认队列，不执行相容性结论 | +| 规则无命中 | 记录"规则缺失/待确认"，不报错 | + +--- + +#### 2.1.5 演示 Demo + +已提供可运行的模拟流水线脚本：[demo_pipeline.py](file:///d:/研究生/ai算法/LabSafe/docs/demo_pipeline.py) + +**运行方式：** + +```bash +python docs/demo_pipeline.py +``` + +**演示场景：** + +| 场景 | 输入 | 预期结果 | +|---|---|---| +| 正常流程 | 乙醇与丙酮同柜相邻 | 命中规则 `rule-flammable-flammable-001`，严重度 `high`，要求人工复核 | +| 质量拦截 | 模糊图片 | 状态 `needs_retake`，返回补拍提示"请保持手机稳定，对焦后拍摄" | +| 无规则命中 | 单独存放的硫酸 | 无规则命中，严重度 `low` | + +**Demo 输出示例（场景 1）：** + +```json +{ + "status": "succeeded", + "entities": [ + {"standard_name": "乙醇", "hazard_class": "易燃液体", "match_type": "exact"}, + {"standard_name": "丙酮", "hazard_class": "易燃液体", "match_type": "exact"} + ], + "rule_hits": [{ + "rule_id": "rule-flammable-flammable-001", + "explanation": "乙醇和丙酮均为易燃液体，同柜可见相邻存放增加火灾风险", + "severity_suggestion": "high", + "requires_review": true + }], + "severity_summary": {"overall": "high", "confidence_weighted_score": 0.86} +} +``` + +### 2.2 数据采集与标注 + +| 任务 | 说明 | 参考文档 | +|---|---|---| +| 采集规范制定 | 按模板逐项拍摄，记录实验室、区域/柜号、时间、采集人 | 06 | +| 标注规范制定 | 目标框、OCR 文字框、风险三态（risk / safe_observed / cannot_determine） | 06 | +| 困难样本池 | 建立 300~500 个反光、遮挡、小目标、模糊样本池 | 06 | +| 数据切分 | 按实验室、采集批次和场景分组切分 70/15/15，测试集冻结后不回流训练 | 06 | +| 人脸脱敏 | 训练前对非必要人脸自动打码或裁剪 | 06 | + +### 2.3 评测与质量保障 + +| 任务 | 说明 | 参考文档 | +|---|---|---| +| 分阶段评测集 | 第 1-3 天冻结最小基线集，第 4-7 天临时门槛，第 8-10 天冻结候选验收集 | 02、14 | +| 模型指标 | 检测：mAP、召回率；OCR：字符/字段准确率；风险：高风险召回率、误报率、无法判断率 | 06 | +| 版本追踪 | 模型版本、阈值、数据集版本、代码提交号写入 `InferenceRun` | 06 | +| 困难样本导出 | 支持批量困难样本导出用于迭代 | 14 | + +### 2.4 推理服务工程化 + +| 任务 | 说明 | 参考文档 | +|---|---|---| +| 独立进程/服务 | AI 推理作为独立进程运行，通过版本化 HTTP/gRPC 契约与业务层通信 | 03、14 | +| 异步任务接入 | 接入 Celery + Redis 队列，支持 `image_quality`、`detection`、`ocr`、`entity_normalization`、`compatibility_check` 任务类型 | 03 | +| 幂等与重试 | 每个任务具备幂等键、最大重试次数、退避策略、死信记录和人工重放入口 | 14 | +| 超时控制 | 质量检查 P95 ≤5 秒；GPU 完整推理 P95 ≤30 秒；CPU 降级允许异步超时 | 14 | +| 资源隔离 | GPU 资源隔离、模型热更新、独立扩缩容和模型版本追踪 | 14 | +| 不可覆盖记录 | 推理结果写入不可覆盖的版本化记录 | 03 | + +### 2.5 推理端交付物 + +- 可运行的推理服务进程（FastAPI/gRPC） +- 质量检测、检测、OCR、标准化、规则匹配各模块基线模型 +- 化学品实体表与规则集（≥20 条） +- 评测脚本与冻结测试集 +- 困难样本池与迭代报告 +- 模型/规则版本登记与回滚方案 + +--- + +## 三、业务端要做的事 + +### 3.1 核心领域服务 + +| 模块 | 任务 | 参考文档 | +|---|---|---| +| 认证与权限 | 登录、角色、资源范围、会话管理；支持租户（学校）-学院-实验室层级 | 14 | +| 实验室管理 | 实验室建档、归档、授权；区域、货架/柜号管理 | 04、14 | +| 模板管理 | 巡检模板发布、停用、复制；检查项定义 | 04 | +| 巡检管理 | 创建巡检、检查项、图片元数据、采集状态流转 | 04、14 | +| 复核服务 | Finding 确认、修改、驳回、无法判断；保存复核记录和原因 | 04、05 | +| 整改服务 | 派发、接收、提交证据、复查、销项；逾期派生状态 | 04、14 | +| 报表服务 | 聚合统计、CSV/PDF 导出、证据索引 | 04 | +| 审计服务 | 全量状态变更记录、操作者、时间、前后值 | 04、14 | + +### 3.2 异步任务编排 + +| 任务 | 说明 | 参考文档 | +|---|---|---| +| 队列管理 | Celery + Redis 队列搭建，任务状态追踪（queued/running/succeeded/failed/retrying/cancelled） | 05 | +| 任务编排 | 上传后按序触发质量检测 → 检测 → OCR → 标准化 → 规则检查 | 03 | +| 状态机同步 | 推理完成后将结果写回业务层，驱动 InspectionItem 和 Finding 状态流转 | 14 | +| 失败处理 | 质量失败 → `needs_retake`；推理失败 → 稳定错误码 + 重试入口 | 05 | + +### 3.3 API 层 + +| 任务 | 说明 | 参考文档 | +|---|---|---| +| REST API | 实现 `/api/v1` 下全部端点（认证、实验室、模板、巡检、上传、推理结果、复核、整改、规则、报表、审计） | 05 | +| 幂等设计 | 写操作要求 `Idempotency-Key` | 05 | +| 权限过滤 | 所有资源按权限过滤，禁止依赖前端隐藏按钮 | 05 | +| 错误格式 | 统一 `{code, message, details, request_id}` | 05 | +| 领域命令 | 状态变化只能通过领域命令接口，禁止通用 `PATCH status` | 05 | + +### 3.4 前端（Web + 移动端） + +| 端 | 任务 | 参考文档 | +|---|---|---| +| 移动巡检端 | 登录、选择实验室/模板、检查项列表、拍摄/上传、质量提示、AI 结果复核、整改任务、复查拍摄 | 07 | +| Web 管理端 | 总览看板、实验室/人员/权限管理、巡检记录、风险问题、整改任务、规则与模型版本、报表导出、审计日志 | 07 | +| 交互规范 | 检查项状态显示（未开始/处理中/待复核/已完成/需补拍）；确认/驳回/无法判断/补拍为明确动作 | 07 | +| 错误处理 | 上传失败保留本地草稿；推理失败显示稳定错误码；无规则匹配显示"规则缺失/待确认" | 07 | +| 响应式 | 手机单手可达；处理中状态可离开页面异步等待 | 07 | + +### 3.5 基础设施 + +| 任务 | 说明 | 参考文档 | +|---|---|---| +| 数据库 | MySQL 部署，核心业务表设计（含 `tenant_id`），JSON 字段证据存储 | 04、14 | +| 对象存储 | MinIO 部署，图片和报告存储，上传凭证签发 | 03、14 | +| 缓存/队列 | Redis 部署，Celery 队列和短期状态 | 03 | +| 认证 | 本地账号体系（首期），预留 OIDC/SAML 接口 | 14 | +| 可观测性 | 结构化日志 + OpenTelemetry + Prometheus；记录请求 ID、任务 ID、模型/规则版本、耗时 | 03 | +| 备份恢复 | RPO ≤24 小时，RTO ≤4 小时 | 14 | + +### 3.6 业务端交付物 + +- 可运行的业务 API 服务（FastAPI） +- 响应式 Web 应用（移动巡检端 + Web 管理端） +- MySQL 数据库 Schema 与迁移脚本 +- MinIO + Redis + Celery 基础设施 +- 权限、审计、状态机核心逻辑 +- 异步任务编排与监控 +- 断网 Demo 环境（预置数据、模型、规则、账号） + +--- + +## 四、业务端与推理端协作接口 + +| 接口 | 方向 | 内容 | 参考文档 | +|---|---|---|---| +| 上传触发 | 业务 → 推理 | 图片写入 MinIO 后，业务层创建异步推理任务 | 03 | +| 推理结果回传 | 推理 → 业务 | 版本化事实（质量、检测框、OCR 字段、标准化实体、规则命中）写入 `InferenceRun` | 03、04 | +| 人工复核 | 业务 → 推理 | 复核修改（字段修正）作为新输入重新触发推理（可选） | 05 | +| 规则发布 | 推理 → 业务 | 新规则版本发布事件 `RuleVersionPublished`，业务层用于新任务 | 14 | +| 质量拦截 | 推理 → 业务 | 质量检查失败直接返回 `needs_retake`，不进入后续模型 | 03 | + +--- + +## 五、里程碑对照 + +| 里程碑 | 推理端关键交付 | 业务端关键交付 | +|---|---|---| +| M1 | 采集/标注规范确认；基线模型选型 | 合作实验室确认；验收口径冻结 | +| M2 | 首轮采集、标注；困难样本池初版 | MySQL/MinIO/Redis/API 骨架；模型登记 | +| M3 | 实体标准化；20+ 规则；测试集冻结 | 推理结果回传契约；复核界面初版 | +| M4 | 模型优化（反光/倾斜/小目标/遮挡） | 巡检-复核-整改-复查-销项闭环；异步推理接入 | +| M5 | 困难样本迭代；指标达标验证 | 端到端联调；性能/异常/权限测试；试运行 | +| M6 | 算法测试报告；模型/规则版本归档 | 业务对照报告；部署文档；Demo 脚本 | + +--- + +## 六、不可妥协的边界 + +1. **AI 不输出最终安全结论** — 只输出事实和候选，必须人工确认 +2. **规则不由大模型生成** — 首期不接入 LLM，规则由专家批准 +3. **历史记录不可改写** — 规则/模型版本不可变，历史记录绑定原版本 +4. **状态变更必须审计** — 所有状态转移走事务 + 审计事件 +5. **权限不依赖前端** — 后端全量过滤，前端只做展示 diff --git a/picture/generate_test_images.py b/picture/generate_test_images.py new file mode 100644 index 0000000..f8afde6 --- /dev/null +++ b/picture/generate_test_images.py @@ -0,0 +1,265 @@ +# 生成更真实的测试图片用于 LabSafe Demo +# 运行: python generate_test_images.py + +from PIL import Image, ImageDraw, ImageFont, ImageFilter +import os +import math + + +def cv2_imread_unicode(path): + import cv2 + import numpy as np + return cv2.imdecode(np.fromfile(path, dtype=np.uint8), cv2.IMREAD_COLOR) + + +def draw_realistic_bottle(draw, img, cx, cy, width, height, liquid_color, label_text, font, small_font): + """绘制更真实的试剂瓶（圆柱体 + 渐变 + 阴影 + 标签 + 瓶盖）""" + x = cx - width // 2 + top = cy - height // 2 + bottom = cy + height // 2 + + # 1. 阴影（投射在货架上） + shadow_offset = 8 + for i in range(width // 2): + alpha = int(80 * (1 - i / (width / 2))) + draw.ellipse( + [x + i, bottom + shadow_offset, x + width - i, bottom + shadow_offset + 6], + fill=(max(0, 139 - alpha), max(0, 90 - alpha), max(0, 43 - alpha)) + ) + + # 2. 瓶身渐变（左暗右亮模拟圆柱光照） + body_left = x + 4 + body_top = top + 25 # 瓶盖下方 + body_right = x + width - 4 + body_bottom = bottom - 5 + + # 用多条竖线模拟渐变 + for px in range(body_left, body_right): + t = (px - body_left) / (body_right - body_left) + # 中间最亮，两侧暗（圆柱光照模型） + brightness = 1.0 - abs(t - 0.4) * 1.2 + brightness = max(0.3, min(1.0, brightness)) + r = int(liquid_color[0] * brightness + 40) + g = int(liquid_color[1] * brightness + 40) + b = int(liquid_color[2] * brightness + 40) + r, g, b = min(r, 255), min(g, 255), min(b, 255) + draw.line([(px, body_top), (px, body_bottom)], fill=(r, g, b)) + + # 瓶身边框 + draw.rectangle([body_left, body_top, body_right, body_bottom], + outline=(60, 60, 60), width=2) + + # 3. 瓶肩（圆弧过渡） + shoulder_h = 15 + for i in range(shoulder_h): + t = i / shoulder_h + offset = int((1 - t) * width * 0.15) + draw.line([(x + 4 + offset, body_top - shoulder_h + i), + (x + width - 4 - offset, body_top - shoulder_h + i)], + fill=(int(liquid_color[0] * 0.6 + 30), int(liquid_color[1] * 0.6 + 30), int(liquid_color[2] * 0.6 + 30))) + + # 4. 瓶颈 + neck_w = width // 3 + neck_x = x + (width - neck_w) // 2 + neck_top = top + 8 + draw.rectangle([neck_x, neck_top, neck_x + neck_w, body_top - shoulder_h + 2], + fill=(int(liquid_color[0] * 0.7), int(liquid_color[1] * 0.7), int(liquid_color[2] * 0.7)), + outline=(60, 60, 60), width=1) + + # 5. 瓶盖 + cap_w = neck_w + 6 + cap_x = x + (width - cap_w) // 2 + cap_top = top + cap_bottom = top + 12 + # 瓶盖渐变 + for px in range(cap_x, cap_x + cap_w): + t = (px - cap_x) / cap_w + brightness = 1.0 - abs(t - 0.5) * 0.8 + r = int(200 * brightness) + g = int(40 * brightness) + b = int(40 * brightness) + draw.line([(px, cap_top), (px, cap_bottom)], fill=(r, g, b)) + draw.rectangle([cap_x, cap_top, cap_x + cap_w, cap_bottom], outline=(120, 20, 20), width=2) + + # 6. 标签（白色矩形 + 文字 + 液位线） + label_w = width - 12 + label_h = 55 + label_x = x + 6 + label_y = body_top + 10 + + # 标签背景（微泛黄） + for py in range(label_y, label_y + label_h): + for px in range(label_x, label_x + label_w): + pass # 已用 rectangle 填充 + + draw.rectangle([label_x, label_y, label_x + label_w, label_y + label_h], + fill=(252, 248, 235), outline=(100, 100, 100), width=1) + + # 标签文字 + draw.text((label_x + 8, label_y + 6), label_text, fill=(20, 20, 20), font=font) + draw.text((label_x + 8, label_y + 32), "2025-01", fill=(80, 80, 80), font=small_font) + + # 7. 高光（左上角白色弧线模拟反光） + highlight_x = body_left + 4 + for py in range(body_top + 5, body_bottom - 5, 2): + for px in range(highlight_x, highlight_x + 6, 2): + offset_y = (py - body_top) / (body_bottom - body_top) + fade = 1.0 - abs(offset_y - 0.3) * 3 + if fade > 0: + alpha = int(60 * fade) + pixel = img.getpixel((px, py)) + new_r = min(255, pixel[0] + alpha) + new_g = min(255, pixel[1] + alpha) + new_b = min(255, pixel[2] + alpha) + img.putpixel((px, py), (new_r, new_g, new_b)) + + +def draw_shelf(draw, w, h): + """绘制货架（多层木板 + 阴影）""" + # 外框 + draw.rectangle([30, 30, w - 30, h - 30], fill=(100, 65, 30), outline=(60, 40, 15), width=3) + # 内壁 + draw.rectangle([40, 40, w - 40, h - 40], fill=(130, 85, 45), outline=(80, 55, 25), width=2) + # 隔板 + shelf_y = h // 2 + 20 + draw.rectangle([40, shelf_y, w - 40, shelf_y + 8], fill=(90, 60, 28), outline=(60, 40, 18), width=1) + # 背景纹理（横纹） + for y in range(42, h - 42, 3): + c = 128 + (y % 6) + draw.line([(42, y), (w - 42, y)], fill=(c, c - 40, c - 60)) + + +def create_normal_scene(): + """场景1: 乙醇与丙酮相邻存放（更真实的试剂瓶）""" + w, h = 640, 480 + img = Image.new('RGB', (w, h), (240, 235, 225)) + draw = ImageDraw.Draw(img) + + try: + font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 20) + small_font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 14) + except: + font = ImageFont.load_default() + small_font = font + + draw_shelf(draw, w, h) + + # 两个瓶子（位置错开，模拟真实摆放） + draw_realistic_bottle(draw, img, 200, 260, 90, 220, (180, 180, 185), "乙醇", font, small_font) + draw_realistic_bottle(draw, img, 340, 260, 90, 220, (170, 170, 175), "丙酮", font, small_font) + + img = img.filter(ImageFilter.GaussianBlur(radius=0.3)) + return img + + +def create_blurry_image(): + """场景2: 模糊图片""" + w, h = 640, 480 + img = Image.new('RGB', (w, h), (240, 235, 225)) + draw = ImageDraw.Draw(img) + + try: + font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 20) + small_font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 14) + except: + font = ImageFont.load_default() + small_font = font + + draw_shelf(draw, w, h) + draw_realistic_bottle(draw, img, 200, 260, 90, 220, (180, 180, 185), "乙醇", font, small_font) + draw_realistic_bottle(draw, img, 340, 260, 90, 220, (170, 170, 175), "丙酮", font, small_font) + + img = img.filter(ImageFilter.GaussianBlur(radius=6)) + return img + + +def create_single_bottle(): + """场景3: 单独存放的硫酸""" + w, h = 640, 480 + img = Image.new('RGB', (w, h), (240, 235, 225)) + draw = ImageDraw.Draw(img) + + try: + font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 20) + small_font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 14) + except: + font = ImageFont.load_default() + small_font = font + + draw_shelf(draw, w, h) + draw_realistic_bottle(draw, img, 320, 260, 95, 230, (200, 180, 100), "硫酸", font, small_font) + + img = img.filter(ImageFilter.GaussianBlur(radius=0.3)) + return img + + +def create_glare_image(): + """场景4: 反光图片""" + w, h = 640, 480 + img = Image.new('RGB', (w, h), (240, 235, 225)) + draw = ImageDraw.Draw(img) + + try: + font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 20) + small_font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 14) + except: + font = ImageFont.load_default() + small_font = font + + draw_shelf(draw, w, h) + draw_realistic_bottle(draw, img, 200, 260, 90, 220, (180, 180, 185), "乙醇", font, small_font) + + # 大面积白色高光 + for i in range(200): + x = 150 + i + y = 100 + i // 2 + r = 255 + g = 255 + b = 255 + draw.ellipse([x - 60, y - 40, x + 60, y + 40], fill=(r, g, b, 180)) + + return img + + +def create_dark_image(): + """场景5: 光线不足""" + w, h = 640, 480 + img = Image.new('RGB', (w, h), (240, 235, 225)) + draw = ImageDraw.Draw(img) + + try: + font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 20) + small_font = ImageFont.truetype("C:/Windows/Fonts/simhei.ttf", 14) + except: + font = ImageFont.load_default() + small_font = font + + draw_shelf(draw, w, h) + draw_realistic_bottle(draw, img, 200, 260, 90, 220, (180, 180, 185), "乙醇", font, small_font) + draw_realistic_bottle(draw, img, 340, 260, 90, 220, (170, 170, 175), "丙酮", font, small_font) + + # 整体变暗 + import numpy as np + arr = np.array(img) + arr = (arr * 0.25).astype(np.uint8) + img = Image.fromarray(arr) + return img + + +if __name__ == "__main__": + output_dir = os.path.dirname(os.path.abspath(__file__)) + + images = { + "shelf_normal.jpg": create_normal_scene(), + "shelf_blurry.jpg": create_blurry_image(), + "shelf_single.jpg": create_single_bottle(), + "shelf_glare.jpg": create_glare_image(), + "shelf_dark.jpg": create_dark_image(), + } + + for filename, img in images.items(): + filepath = os.path.join(output_dir, filename) + img.save(filepath, "JPEG", quality=92) + print(f"生成: {filepath}") + + print(f"\n共生成 {len(images)} 张测试图片") diff --git a/picture/real_bus.jpg b/picture/real_bus.jpg new file mode 100644 index 0000000..b43e311 Binary files /dev/null and b/picture/real_bus.jpg differ diff --git a/picture/real_zidane.jpg b/picture/real_zidane.jpg new file mode 100644 index 0000000..92d72ea Binary files /dev/null and b/picture/real_zidane.jpg differ diff --git a/picture/shelf_blurry.jpg b/picture/shelf_blurry.jpg new file mode 100644 index 0000000..b2761ca Binary files /dev/null and b/picture/shelf_blurry.jpg differ diff --git a/picture/shelf_dark.jpg b/picture/shelf_dark.jpg new file mode 100644 index 0000000..7057df5 Binary files /dev/null and b/picture/shelf_dark.jpg differ diff --git a/picture/shelf_glare.jpg b/picture/shelf_glare.jpg new file mode 100644 index 0000000..bd3a989 Binary files /dev/null and b/picture/shelf_glare.jpg differ diff --git a/picture/shelf_normal.jpg b/picture/shelf_normal.jpg new file mode 100644 index 0000000..804dd49 Binary files /dev/null and b/picture/shelf_normal.jpg differ diff --git a/picture/shelf_single.jpg b/picture/shelf_single.jpg new file mode 100644 index 0000000..1f3df28 Binary files /dev/null and b/picture/shelf_single.jpg differ