From a27256a23ba4208a6c9118e77879e09dc0fef9f2 Mon Sep 17 00:00:00 2001 From: wulu Date: Wed, 23 Sep 2026 07:18:31 +0800 Subject: [PATCH 01/15] feat(mosaico): add local camera gestures Add swipe navigation, wake-only screensaver input, and held OK clock entry. Keep recognition local and disabled by default in display settings. Expand OTA slots to 7 MiB to embed both models with the application. Existing 4 MiB layouts require a first wired partition migration; segmented installation preserves NVS but replaces the old fallback slot. Validation: enabled/disabled firmware and test images build; 31 host Unity tests pass with ASan/UBSan; Bridge tests pass (620, 7 skipped). Camera behavior, performance, and rollback await hardware validation. --- THIRD_PARTY_LICENSES.md | 56 +++++ .../sources/firmware_release.py | 3 +- bridge/src/quotaframe_bridge/targets.py | 2 +- bridge/tests/test_firmware_release.py | 13 +- docs/design/mosaico-gesture-input.md | 175 ++++++++++++++ docs/roadmap.md | 1 + docs/validation/mosaico-gestures.md | 77 ++++++ .../esp_desktop_buddy_folder_push/Kconfig | 2 +- .../esp_desktop_buddy_folder_push/UPSTREAM.md | 4 + .../usage_panel_state/screensaver_state.hpp | 2 + .../usage_panel_state/screensaver_state.cpp | 5 + .../targets/esp_mosaico/CAMERA_UPSTREAM.md | 47 ++++ firmware/targets/esp_mosaico/README.md | 10 +- .../targets/esp_mosaico/dependencies.lock | 192 ++++++++++++++- .../targets/esp_mosaico/main/CMakeLists.txt | 7 + .../esp_mosaico/main/Kconfig.projbuild | 28 +++ .../targets/esp_mosaico/main/app_main.cpp | 66 +++++- .../targets/esp_mosaico/main/display_ui.cpp | 71 +++++- .../targets/esp_mosaico/main/display_ui.hpp | 12 +- .../esp_mosaico/main/gesture_camera.cpp | 184 +++++++++++++++ .../esp_mosaico/main/gesture_camera.hpp | 32 +++ .../esp_mosaico/main/gesture_image.cpp | 44 ++++ .../esp_mosaico/main/gesture_image.hpp | 19 ++ .../esp_mosaico/main/gesture_input.cpp | 223 ++++++++++++++++++ .../esp_mosaico/main/gesture_input.hpp | 40 ++++ .../esp_mosaico/main/gesture_tracker.cpp | 99 ++++++++ .../esp_mosaico/main/gesture_tracker.hpp | 53 +++++ .../esp_mosaico/main/idf_component.yml | 3 + .../esp_mosaico/main/panel_settings.cpp | 40 ++++ .../esp_mosaico/main/panel_settings.hpp | 16 ++ firmware/targets/esp_mosaico/partitions.csv | 4 +- .../targets/esp_mosaico/sdkconfig.defaults | 28 ++- .../test_apps/logic/main/CMakeLists.txt | 3 + .../test_apps/logic/main/test_gestures.cpp | 106 +++++++++ .../main/test_screensaver_state.cpp | 14 ++ 35 files changed, 1654 insertions(+), 27 deletions(-) create mode 100644 docs/design/mosaico-gesture-input.md create mode 100644 docs/validation/mosaico-gestures.md create mode 100644 firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md create mode 100644 firmware/targets/esp_mosaico/main/gesture_camera.cpp create mode 100644 firmware/targets/esp_mosaico/main/gesture_camera.hpp create mode 100644 firmware/targets/esp_mosaico/main/gesture_image.cpp create mode 100644 firmware/targets/esp_mosaico/main/gesture_image.hpp create mode 100644 firmware/targets/esp_mosaico/main/gesture_input.cpp create mode 100644 firmware/targets/esp_mosaico/main/gesture_input.hpp create mode 100644 firmware/targets/esp_mosaico/main/gesture_tracker.cpp create mode 100644 firmware/targets/esp_mosaico/main/gesture_tracker.hpp create mode 100644 firmware/targets/esp_mosaico/main/panel_settings.cpp create mode 100644 firmware/targets/esp_mosaico/main/panel_settings.hpp create mode 100644 firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index a6f42ab..7c2f032 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -566,3 +566,59 @@ SOFTWARE. - Source: https://components.espressif.com/components/tangyumei3535/bmm150_sensorapi/versions/1.0.0 - SPDX conclusion from distributed license: `BSD-3-Clause` - Copyright/notice: https://components.espressif.com/components/tangyumei3535/bmm150_sensorapi/versions/1.0.0/license + +## `espressif/esp-dl` + +- ESP-Mosaico version: `3.3.11` +- Source: https://components.espressif.com/components/espressif/esp-dl/versions/3.3.11 +- SPDX conclusion: `MIT` +- Copyright/notice: https://components.espressif.com/components/espressif/esp-dl/versions/3.3.11/license + +## `espressif/esp_video` + +- ESP-Mosaico version: `2.5.0` +- Source: https://components.espressif.com/components/espressif/esp_video/versions/2.5.0 +- SPDX conclusion: `LicenseRef-Espressif-MIT` +- Copyright/notice: https://components.espressif.com/components/espressif/esp_video/versions/2.5.0/license + +## `espressif/esp_cam_sensor` + +- ESP-Mosaico version: `2.6.0` +- Source: https://components.espressif.com/components/espressif/esp_cam_sensor/versions/2.6.0 +- SPDX conclusion: `Apache-2.0` +- Copyright/notice: https://components.espressif.com/components/espressif/esp_cam_sensor/versions/2.6.0/license + +## `espressif/esp_sccb_intf` + +- ESP-Mosaico version: `0.0.8` +- Source: https://components.espressif.com/components/espressif/esp_sccb_intf/versions/0.0.8 +- SPDX conclusion: `Apache-2.0` +- Copyright/notice: https://components.espressif.com/components/espressif/esp_sccb_intf/versions/0.0.8/license + +## `espressif/dl_fft` + +- ESP-Mosaico version: `0.7.0` +- Source: https://components.espressif.com/components/espressif/dl_fft/versions/0.7.0 +- SPDX conclusion: `MIT` +- Copyright/notice: https://components.espressif.com/components/espressif/dl_fft/versions/0.7.0/license + +## `espressif/usb_host_uvc` + +- ESP-Mosaico version: `2.5.1` +- Source: https://components.espressif.com/components/espressif/usb_host_uvc/versions/2.5.1 +- SPDX conclusion: `Apache-2.0` +- Copyright/notice: https://components.espressif.com/components/espressif/usb_host_uvc/versions/2.5.1/license + +## `espressif/hand_detect` + +- ESP-Mosaico version: `0.2.0` +- Source: https://components.espressif.com/components/espressif/hand_detect/versions/0.2.0 +- SPDX conclusion: `MIT` +- Copyright/notice: https://components.espressif.com/components/espressif/hand_detect/versions/0.2.0/license + +## `espressif/hand_gesture_recognition` + +- ESP-Mosaico version: `0.2.0` +- Source: https://components.espressif.com/components/espressif/hand_gesture_recognition/versions/0.2.0 +- SPDX conclusion: `MIT` +- Copyright/notice: https://components.espressif.com/components/espressif/hand_gesture_recognition/versions/0.2.0/license diff --git a/bridge/src/quotaframe_bridge/sources/firmware_release.py b/bridge/src/quotaframe_bridge/sources/firmware_release.py index bbbe877..a749d0d 100644 --- a/bridge/src/quotaframe_bridge/sources/firmware_release.py +++ b/bridge/src/quotaframe_bridge/sources/firmware_release.py @@ -15,10 +15,11 @@ from quotaframe_bridge.protocol.validation import unique_object, validate_identifier from quotaframe_bridge.release_config import release_repository from quotaframe_bridge.versioning import SemVer, VersionError +from quotaframe_bridge.targets import TARGETS _SHA256 = re.compile(r"^[0-9a-f]{64}$") MAX_MANIFEST_BYTES = 64 * 1024 -MAX_IMAGE_BYTES = 4 * 1024 * 1024 +MAX_IMAGE_BYTES = max(target.ota_partition_bytes for target in TARGETS) class FirmwareReleaseError(ValueError): diff --git a/bridge/src/quotaframe_bridge/targets.py b/bridge/src/quotaframe_bridge/targets.py index 54a27be..822e091 100644 --- a/bridge/src/quotaframe_bridge/targets.py +++ b/bridge/src/quotaframe_bridge/targets.py @@ -102,7 +102,7 @@ def ota_transfer_bytes(self) -> int: full_image="mosaico_usage_panel_full.bin", release_stem="espressif-esp-mosaico", image_chip_id=32, - ota_partition_bytes=4 * 1024 * 1024, + ota_partition_bytes=7 * 1024 * 1024, test_apps=("firmware/targets/esp_mosaico/test_apps/logic",), web_flash=WebFlashDefinition( name="ESP-Mosaico", diff --git a/bridge/tests/test_firmware_release.py b/bridge/tests/test_firmware_release.py index 35b32d6..c9dd724 100644 --- a/bridge/tests/test_firmware_release.py +++ b/bridge/tests/test_firmware_release.py @@ -113,13 +113,24 @@ async def test_rejects_oversized_manifest_image_and_fetch_result(self) -> None: with self.assertRaises(FirmwareReleaseError): await source.download_image(manifest.for_target("m5sticks3")) - too_large = json.loads(self.manifest(size=4 * 1024 * 1024 + 1)) + too_large = json.loads(self.manifest(size=7 * 1024 * 1024 + 1)) source = FirmwareReleaseSource( fetcher=lambda _url, _limit: json.dumps(too_large).encode() ) with self.assertRaises(FirmwareReleaseError): await source.fetch_manifest(self.manifest_url) + async def test_accepts_mosaico_image_within_expanded_ota_slot(self) -> None: + source = FirmwareReleaseSource( + fetcher=lambda _url, _limit: self.manifest( + target="esp_mosaico", firmware_project="mosaico_usage_panel", + size=7 * 1024 * 1024, + ) + ) + manifest = await source.fetch_manifest(self.manifest_url) + image = manifest.for_target("esp_mosaico", "mosaico_usage_panel") + self.assertEqual(image.size, 7 * 1024 * 1024) + async def test_rejects_noncanonical_semver(self) -> None: for version in ("01.2.3", "1.02.3", "1.2.03", "1.2.3-alpha..1", "1.2.3-01"): with self.subTest(version=version): diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md new file mode 100644 index 0000000..a93edaa --- /dev/null +++ b/docs/design/mosaico-gesture-input.md @@ -0,0 +1,175 @@ +# Mosaico 本地手势输入规划 + +状态：软件实现与构建验证完成，待实机验收。2026-09-22 核对；工程基线 `07ac386364ff680bf1835ace646c788975a0cbbb`，规划分支 `codex/mosaico-gesture-plan`。 + +## 目标与范围 + +在 ESP-Mosaico 上增加可选的本地摄像头输入，保留触摸、AI 键、BLE 和自动时钟屏保。先验证 CameraBoard → HandDetect → 横向轨迹 → 翻页/唤醒，再增加现成分类器的 `ok` 手势进入时钟。 + +摄像头图像仅在设备内处理，不上传、不默认保存、不显示常驻预览。第一阶段不训练模型、不引入 ESP-VISION/MicroPython 或 ESP-WHO 整套应用、不扩展 Bridge 协议、不把视觉依赖加入其他板卡。识别区域内出现手不等于用户发出命令。 + +此处“解锁”指退出时钟屏保，不是身份验证，也不涉及芯片深度睡眠唤醒。摄像头唤醒要求采集与识别链路仍在运行，不能据此承诺低功耗。 + +## 两份报告的取舍 + +已阅读分享对话和补充报告。采用分享对话最后确认的“挥手翻页/唤醒”语义；👌 OK 进入时钟作为第二阶段能力。两份报告的方向描述相反，以当前触摸行为为准。 + +| 当前状态 | 有效动作 | 行为 | +| --- | --- | --- | +| 正常用量页 | 向左挥手 | 下一页 | +| 正常用量页 | 向右挥手 | 上一页 | +| 时钟屏保 | 任一有效横向挥手 | 只恢复原页面；本轮不再翻页 | +| 正常用量页 | 稳定 `ok` | 进入时钟并保存原页面 | +| 时钟屏保 | `ok` | 无操作，不执行 Toggle | +| 设置、配对确认、OTA | 任意手势 | 丢弃，不延后执行 | + +左右方向均为用户面对当前屏幕所见的方向。挥动判定只用 HandDetect，因此不能宣称识别了“张掌”；若日常活动误触发无法达标，再评估以 `five` 分类作为挥动的手形门控。 + +补充报告中“看到手/普通运动即可唤醒”暂不采纳，避免桌前活动持续打断屏保。ROI motion 仅是后续功耗优化候选，不能代替有效挥手确认。 + +## 已核对的证据与缺口 + +| 项目 | 当前证据 | 对实施的影响 | +| --- | --- | --- | +| ESP-DL S31 支持 | Model Zoo 列出 S31；HandDetect 与 HandGesture 的 CMake 均把 `esp32s31` 映射到 `models/p4` | 发布组件已在 IDF v6.1 下完整构建通过；真实推理仍待实机验证 | +| 现成动作类别 | 示例列出 `five`、`ok` 等类别，没有独立 `fist`；`no_gesture` 是未定义手势 | 不把 `no_gesture` 当作拳头，不投入自训练 | +| 模型输入与性能 | 检测输入 224×224，分类输入 128×128；子目录耗时表只给 S3/P4 | 不将 P4 耗时外推为 S31 性能，也不将输入大小视为相机输出模式 | +| ESP-VISION | 官方支持页列出 ESP32-S31-MOSAICO；README 指定 S31 使用 IDF v6.1 | 参考 board port、采集和转换；本次未独立确认报告中的 CI 配置细节 | +| ESP-WHO | README 描述采集/推理异步运行；板卡清单未列 Mosaico | 仅作生命周期与任务划分参考 | +| 当前 BSP | `BSP_CAPS_CAMERA` 为 0，已有扩展槽电源/I2C/GPIO 基础，无采集组件依赖 | CameraBoard 接入是首个工程任务，不能只改能力宏 | +| 摄像头参考实现 | 相邻 esp-mosaico-claw 工程有 OV3640 DVP 的 `mosaico_camera`，当前接口支持左槽和帧获取/归还 | 它依赖 `subboard_support`、模块管理、board manager 及 esp-claw 内的 esp_video；不能直接复制一个组件就认为可用 | +| 固件存储 | 目标配置 16 MB Flash；原有两个 OTA app 分区各 4 MiB；本实现改为 7 MiB | 完整应用约 5.16 MiB，首次需有线迁移，见下文 | + +相邻工程核对时 HEAD 为 `903fe11ec3ca4a8881afef104e53a8da040896ac`，摄像头头文件和实现存在未提交修改，README 的默认分辨率/缓冲数量也与当前头文件不同。它只提供参考，实施时必须选定可复现的来源和许可，不能将本地工作区当作已验证的上游版本。 + +## 现有接入点 + +路径均相对仓库根目录。 + +| 路径/符号 | 复用方式 | +| --- | --- | +| `firmware/targets/esp_mosaico/main/app_main.cpp` | 主循环统一拥有页面、屏保、设置、OTA/配对门控；新增手势消费点 | +| `firmware/targets/esp_mosaico/main/orientation.cpp` / `navigation_swipe()` | 现有左滑→Next、右滑→Previous；保留触摸映射，单独校准相机→屏幕坐标 | +| `firmware/components/usage_panel_state/include/usage_panel_state/page_state.hpp` / `page_after_swipe()` | 复用页序与边界行为 | +| `firmware/components/usage_panel_state/screensaver_state.cpp` | 复用原页面保存、活动时间和屏保生命周期；按需要增加明确的输入方法 | +| `firmware/targets/esp_mosaico/main/display_ui.hpp` / `request_page()` | 主循环同步渲染请求，不由推理任务操作 LVGL | +| `firmware/components/usage_ble/include/usage_ble/app_events.hpp` | 当前队列是 BLE 边界，不加入摄像头帧或视觉依赖 | +| `firmware/components/esp-mosaico-bsp/include/bsp/subboard.h` | 复用扩展槽资源；确认 GPIO14 在摄像头模式是 D4，不能同时作 EEPROM 地址选择输出 | +| `firmware/components/esp-mosaico-bsp/UPSTREAM.md` | 保留已有 IDF 约束与 TE 局刷修改；必要的 BSP 修改记录于此 | + +不能直接调用现有 `ScreenPage` 远程事件实现手势：当前远程分支在屏保中会恢复页面并继续翻页，还会关闭设置。手势需要“唤醒即消费”和“设置中忽略”的独立策略。 + +## 最小实现结构 + +```text +CameraBoard / esp_video + → 获取最新帧、格式转换 + → HandDetect + → 屏幕坐标中的单手轨迹 + → SwipeLeft / SwipeRight / EnterClock + → Mosaico 主循环检查状态 + → ScreensaverController / page_after_swipe / ui.request_page +``` + +- 一个视觉 worker 串行完成采集、推理和轨迹更新即可起步；先测量再决定是否拆任务或绑核。模型不在 UI 主循环或 LVGL 锁内运行。 +- 只在 Mosaico 增加相机适配和纯轨迹判定代码，优先放目标目录；只有需要独立管理第三方来源时才抽出组件，不预建跨板卡视觉框架。 +- 相机帧的获取与归还由 worker 拥有。所有成功、失败、停用路径归还帧；停止时先结束当前推理并归还缓冲，再关闭流和释放资源。结果中不保留驱动/模型内部容器的借用引用。 +- 采用有界的小事件通道，初始可只有一个待消费事件；满时丢弃，不阻塞。事件包含动作、时间戳和输入会话代次，过期或上下文已改变的事件丢弃。 +- 主循环在处理最新 BLE/OTA/设置状态后消费手势。屏保、页面或输入许可变化时更新会话代次，取消旧轨迹；关闭功能、旋转、错误恢复也清空待执行动作。 +- 首次挥手若发生在屏保中，只唤醒并同步 `page`、`requested_page` 与状态上报；不能随后落入翻页分支。只在命令被接受时重置活动计时。 +- 第二阶段为进入时钟提供幂等语义，不重复调用 `note_remote_toggle()`。如需公共方法，只增加最小的明确进入/有效输入唤醒接口，保持其他输入行为。 + +### 轨迹与一次性触发 + +使用时间戳而非固定帧数定义窗口；检测帧率下降不应改变动作语义。bbox 中心先转为归一化屏幕坐标，包含相机安装方向、镜像与当前已应用的屏幕旋转。旋转过程中放弃当前动作。 + +状态从 `Idle → Tracking → WaitRelease → Idle`： + +1. 单个可信目标稳定出现才开始跟踪；以位置连续性和 bbox 尺寸约束保持目标关联。 +2. 在限定时间内检查净横向位移、累计路径方向一致性、纵向漂移、置信度与合理 bbox 大小。突然跳变、长时间漏检、模糊的多手关联取消候选。 +3. 完成一次动作只发一个事件，随后等待持续无手且最短冷却已结束再重新接受输入。 +4. 单帧漏检、分类器 `no_hand` 或 worker 暂停不能单独作为“手已离开”的证据。恢复采集后要观察新鲜帧，禁止旧事件回放。 + +以下仅为试验起点，不是已验证的产品参数：横向净位移约画面宽度 25%，动作窗口 150–900 ms，横向位移至少为纵向的 2 倍；连续无手约 300 ms，最短冷却约 500 ms；事件有效期约 300 ms。阈值结合有效帧数、视场、距离和漏检率一起校准。持续可见的手不因冷却超时而重复翻页。 + +## 分阶段工作与完成条件 + +### P0：摄像头与模型可行性 + +- [ ] 核对实物主板版本、OV3640 CameraBoard、左槽安装方向；确认扩展槽电源、共享 I2C 和 GPIO 复用。 +- [ ] 固定 esp_video、esp_cam_sensor、HandDetect/ESP-DL 的可获取版本或提交和模型摘要；核查发布包确实含 S31 路径，而不只检查 master。现有 master manifest 的版本为 hand_detect 0.2.0、ESP-DL `~3.3.0`，不能因此认定任意同版本包都含相同支持。 +- [ ] 在现有 BSP 上接入最小采集，避免带入整个模块管理/应用调度框架；如必须复用其一部分，记录具体资源所有权和依赖理由。 +- [ ] 获取实际协商的像素格式、宽高、stride 和缓冲数量；验证 RGB565 字节序或 UYVY 转换、缩放/裁剪和 bbox 坐标。不要直接把 UYVY 指针作为 RGB 图像输入。 +- [ ] 先运行静态图 HandDetect，再接实时帧；确认推理结果和帧归还；在现有 UI、BLE、IMU、电量轮询共存时采集耗时和内存。 +- [ ] 核算最终 app 大小、模型大小、内部 RAM/PSRAM 低水位及连续块。224×224 RGB888 输入约 147 KiB；640×480 双缓冲 RGB565 约 1.17 MiB，仅是缓冲示例，不含张量、显示、驱动和栈，也不表示该格式已被硬件接受。 + +存储决策：模型内嵌 app，使模型随 A/B OTA 和回滚一起切换。裁减未使用的 ESP-DL 像素转换后，完整应用仍约 5.16 MiB，超过旧 4 MiB 槽；采用 7 MiB 双槽，保留 NVS、PHY 与 otadata 地址。首次必须分段有线写入新分区表、应用及初始 OTA 元数据；旧 OTA 不能完成此迁移，旧第二槽不能作为迁移后的回滚来源。具体地址和恢复步骤见[首次烧录与验收](../validation/mosaico-gestures.md)。 + +**完成条件：**可复现的依赖与构建、可用的实时 bbox、资源与延迟记录、正常停止/缺摄像头降级。只构建成功不算实时识别通过。当前已完成软件接入及 UI，以便一次烧录完成端到端验收；实时识别和资源指标未通过实机验证前，不视为产品验收完成。 + +### P1：左右挥手翻页与屏保唤醒 + +- [ ] 完成纯轨迹判定和坐标映射；原型先验证固定朝向，再验证四个朝向。 +- [ ] 接入主循环的 Swipe/Wake 策略、一次性消费、会话失效与屏保计时。 +- [ ] 默认关闭；开发阶段先用 Mosaico 专属编译开关。交付前提供用户可操作的开启/关闭入口，并复用现有设置保存/取消语义。 +- [ ] 无摄像头、采集超时、推理失败、内存不足时禁用视觉输入，保留触摸/按键/BLE，不以 fatal error 重启整个面板。初版不承诺热插拔自动恢复，可在重新启用时重试。 +- [ ] OTA 和配对期间暂停视觉工作，设置期间禁止动作；恢复后清空轨迹并重新确认手离开。处理进行中的 DMA/推理所有权后再停止。 + +**完成条件：**下表的软件与实机验收通过；确认 HandDetect-only 能满足误触发目标，否则先评估 `five` 门控，再决定可用范围。无需等待 👌 OK 功能即可独立评审本阶段。 + +### P2：👌 OK 主动进入时钟 + +- [ ] 引入 HandGestureRecognizer；只对目标手分类，记录新增推理耗时与资源，重新检查 OTA 大小。 +- [ ] 稳定 `ok` 约 500 ms 作为初始停留确认值，增加简洁确认反馈；分数阈值由实测决定，不能把 top-1 输出直接视为可信命令。 +- [ ] 进入时钟后等待手离开，防止自然松手立即唤醒；保持 `ok` 只进入一次。`no_gesture` / `no_hand` 不映射到命令。 +- [ ] 若 P1 误触发需要 `five`，比较增加分类前后的误触、动作遗漏与延迟；不要求先静止半秒才能挥动。 + +**完成条件：**正常页一次 👌 OK 进入、重复保持无副作用、自然松手不退出，随后独立挥手只恢复原页；开关关闭后不再采集。 + +## 验证与决策门槛 + +以下数量和延迟是建议的首轮验收目标，P0 测量后再确认可实现范围，不是当前性能承诺。 + +| 范围 | 检查与建议门槛 | +| --- | --- | +| 纯逻辑 | 在现有 Mosaico logic 测试入口增加少量有意义的轨迹用例：左右、抖动/纵向、漏检/目标跳变、持续停留、冷却/离开、旋转/上下文失效；不模拟整套相机驱动 | +| 屏保 | 若共享控制器有改动，扩展现有 panel_state 测试：首挥只唤醒、恢复原页、重复进入幂等、无效检测不延长超时；保留现有触摸/按键/远程用例 | +| 编译 | 手势开/关均构建；共享状态若变化，构建受影响目标。检查锁文件、镜像大小、分区和生成配置；测试镜像构建与测试执行分别记录 | +| 方向/识别 | 记录实际距离、光线与板方向；每朝向每方向 20 次，有效操作成功率目标 ≥90%，零反向，零单轮多次触发 | +| 唤醒/门控 | 每个用量页进入屏保再挥手，第一次只恢复；设置/配对/OTA 中持续挥手及恢复后均无遗留动作 | +| 误触 | 打字、拿杯、路人、静止手、照明变化等各类场景，合计至少 30 分钟无误触；不能将短期结果宣传为长期误触率 | +| 延迟/共存 | 从动作满足条件到 UI 响应 p95 目标 ≤300 ms；同时记录采集、转换、推理、主循环等待及动作总时长。与关闭功能时对比 UI 帧耗时和 BLE 更新延迟，恶化超过约 20% 时先优化/降频，不默认发布 | +| 资源/恢复 | 连续运行至少 30 分钟，堆低水位趋于稳定、无缓冲耗尽/看门狗；反复启停，缺摄像头与超时故障均可降级 | +| 功耗 | 同亮度、同 BLE 负载测关闭/用量页启用/屏保启用的电流；低频采集是否节能必须测量，S31/P4 的计算能力不能替代功耗证据 | + +固件构建前先运行 `eim --version` 和 `eim list`，按目标注册表使用 v6.1： + +```sh +eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 +eim run "idf.py -C firmware/targets/esp_mosaico/test_apps/logic build" v6.1 +eim run "idf.py -C firmware/test_apps/panel_state build" v6.1 +``` + +烧录、清除存储、发布另行按仓库授权规则执行。 + +## 当前实现与验证记录 + +上述阶段的复选框保留为端到端验收条件，未勾选不等于没有软件实现。已完成： + +- 固定 registry 依赖和模型包；左槽 OV3640 采集适配、双缓冲、UYVY 转 RGB888 和四向映射；来源见 [CAMERA_UPSTREAM.md](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)。 +- 单手轨迹、OK 500 ms 停留、离开后再触发、有界事件与上下文失效；主循环实现翻页、只唤醒、幂等进入时钟及受保护状态屏蔽。 +- 本地设置开关默认关闭，保存/取消，停用释放采集与模型，错误反馈与手动重试。ESP-DL 内部部分分配使用断言；资源预检查降低风险，但不保证所有运行时内存不足均能无重启降级。 +- 7 MiB 双 OTA 分区、Folder Push 上限和 Bridge 容量配置；无共享模型分区。 +- 手势开/关固件、Mosaico logic、共享 panel_state 测试镜像以及 AMOLED 2.16 固件构建通过。31 个手势/屏保主机 Unity 测试通过 ASan/UBSan；Bridge 620 项测试通过，7 项跳过。测试镜像尚未在设备运行。 + +下一步是按[首次烧录与实机验收](../validation/mosaico-gestures.md)安装并开启手势，检查摄像头方向、真实模型输出、UI 提示布局、推理耗时、误触、启停与功耗。未烧录、未发布，也未验证物理 A/B 回滚。不能把本次软件验证解释为 P0/P1/P2 的硬件验收通过。 + +## 参考来源 + +- [分享报告及交互修订](https://chatgpt.com/share/6ab29265-faa8-83e8-8e32-fa2bb8b8989b)；另一份为本次任务内粘贴的补充报告。 +- [ESP-DL Model Zoo](https://github.com/espressif/esp-dl/blob/master/models/README.md) +- [HandDetect CMake](https://github.com/espressif/esp-dl/blob/master/models/hand_detect/CMakeLists.txt) 与 [模型说明](https://github.com/espressif/esp-dl/blob/master/models/hand_detect/README.md) +- [HandGesture CMake](https://github.com/espressif/esp-dl/blob/master/models/hand_gesture_recognition/CMakeLists.txt)、[接口定义](https://github.com/espressif/esp-dl/blob/master/models/hand_gesture_recognition/hand_gesture_recognition.hpp) 与 [类别清单](https://github.com/espressif/esp-dl/blob/master/examples/hand_gesture_recognition/README.md) +- [ESP-VISION 支持板卡](https://docs.espressif.com/projects/esp-vision/en/latest/esp32p4/target-support/index.html) 与 [工程说明](https://github.com/espressif/esp-vision) +- [ESP-WHO](https://github.com/espressif/esp-who) +- [Mosaico 当前移植说明](../../firmware/targets/esp_mosaico/README.md)、[BSP 来源与本地修改](../../firmware/components/esp-mosaico-bsp/UPSTREAM.md)、[软件与硬件验收边界](../validation/software-acceptance.md) diff --git a/docs/roadmap.md b/docs/roadmap.md index e7fd892..abb0cbc 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -19,6 +19,7 @@ | [AMOLED 性能参考](validation/amoled-performance.md) | 内存、刷新耗时与测量条件 | | [ePaper 刷新参考](validation/epaper-refresh.md) | 局刷数据及光学验证边界 | | [接入架构](design/open-device-access.md) | 状态归属、连接与更新生命周期 | +| [Mosaico 手势输入规划](design/mosaico-gesture-input.md) | 摄像头、挥手翻页/唤醒与分阶段验收 | | [接入协议](../protocol/open-device-access.md) | 设备线上字段与校验约束 | | [硬件移植](PORTING.md) | 新目标目录与硬件验收 | | [设计语言](design/amoled-ui.md) | AMOLED 界面规范 | diff --git a/docs/validation/mosaico-gestures.md b/docs/validation/mosaico-gestures.md new file mode 100644 index 0000000..87a1342 --- /dev/null +++ b/docs/validation/mosaico-gestures.md @@ -0,0 +1,77 @@ +# Mosaico 手势固件：首次烧录与验收 + +本功能在设备本地运行：左挥下一页、右挥上一页；时钟中首次挥手只恢复原页面；👌 OK 稳定停留约 500 ms 进入时钟。一轮操作最多触发一次，需要手离开后再操作。触摸和 AI 键继续可用。 + +这是供实机验收的实现。软件测试与固件构建不能证明相机成像、方向、识别率、功耗或长期稳定性；这些仍需在目标设备验证。 + +## 首次安装的分区变化 + +两款模型与固件共同超过原有 4 MiB 应用槽。新布局使用 7 MiB 双 OTA 槽，模型内嵌，支持代码与模型一起更新和回滚。 + +| 分区 | 原布局 | 新布局 | +| --- | --- | --- | +| NVS | `0x9000` / `0x6000` | 保持 | +| PHY | `0xf000` / `0x1000` | 保持 | +| ota_0 | `0x10000` / `0x400000` | `0x10000` / `0x700000` | +| otadata | `0x810000` / `0x2000` | 保持 | +| ota_1 | `0x410000` / `0x400000` | `0x820000` / `0x700000` | + +第二槽结束于 `0xf20000`，在 16 MiB Flash 内；`0x710000` 到 `0x810000` 的空隙保留现有 otadata 地址。Folder Push 限额同步为 7 MiB + 512 字节，目标注册表和 Bridge 下载上限同步更新。 + +**旧 4 MiB 分区不能通过应用 OTA 安装本镜像。首次必须通过有线写入新分区表、应用和初始 OTA 元数据。**设备原有容量检查会拒绝超大镜像；不要绕过检查。现有已安装 Bridge 若仍使用 4 MiB 下载上限，也不能用于传送新镜像。 + +优先使用构建生成的分段烧录参数，保持 NVS 扇区不被覆盖。合并的 full 镜像包含分区间填充，整段写入可能覆盖 NVS，不能用它声称保留配对/设置。本次实现不自动烧录或擦除任何设备。 + +## 从源码准备与烧录 + +在仓库根目录： + +```sh +eim --version +eim list +eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 +``` + +确认生成配置包含 `CONFIG_MOSAICO_GESTURE_INPUT=y`、`CONFIG_CAMERA_OV3640=y`、640×480 UYVY 模式，`CONFIG_ESP_DESKTOP_BUDDY_FOLDER_PUSH_MAX_TRANSFER_BYTES=7340544`，以及 `QF-Mosaico-` 广播前缀。旧生成配置可能保留不同前缀，不能只检查 defaults。构建产物中的 `flash_args` 是本次镜像的地址来源。 + +设备接入左侧 OV3640 CameraBoard。先在设备设置中关闭手势，或断电安装相机板；使用真实串口替换下述 `PORT`。手动进入下载模式后，在 **build 目录**运行： + +```sh +python -m esptool --chip esp32s31 --port PORT --baud 460800 \ + --before no-reset --after no-reset --no-stub write-flash @flash_args +``` + +该命令需要已启用的 ESP-IDF v6.1 Python 环境。等待逐段校验成功后手动重启。它写入 bootloader、分区表、ota_0 和 `ota_data_initial.bin`；不包含 NVS，也不执行 `erase-flash`。原有第二槽内容被新的大应用覆盖，首次迁移后不能依赖旧第二槽自动回滚。 + +如需恢复旧版，先准备旧版的完整、匹配的分段文件（旧分区表、旧应用、bootloader、初始 OTA 元数据），再次进入下载模式写入它们。不要单独写回旧分区表，也不要省略重置 OTA 元数据；旧配对和显示设置所在 NVS 地址未变。未来在新布局内进行正常 A/B OTA 时，无需重复迁移分区。 + +## 启用与操作 + +1. 长按现有显示设置入口，选择 `HAND GESTURES / ON`，点击 `SAVE`。默认关闭；取消不会启用摄像头。 +2. 等待 `CAMERA STARTING` 消失，先把手移出画面至少约半秒，再开始动作。 +3. 正常页左右挥动；一次只翻一页。屏保中第一次挥动只恢复原页。 +4. 正常页做 👌 OK 并短暂停留；进度条完成才进入时钟。进入后先移开手，下一轮挥手再唤醒。 +5. 设置、配对确认和 OTA 期间不接受手势。关闭并保存后释放采集和模型资源。 + +`CHECK LEFT CAMERA` 表示采集未能启动或连续失败。检查左槽相机，关闭并保存，再开启并保存可重试。`LOW MEMORY` 表示资源预检查未通过；`RESTART DEVICE` 表示驱动资源未能安全释放，需要重启后再试。 + +摄像头占用 USB Serial/JTAG 的 GPIO33；运行期间使用 UART 获取诊断。日志只报告动作、周期时间、内存和栈余量，不输出图像或用户数据。屏保时摄像头仍工作，因此本功能不是深睡眠唤醒，也未验证续航收益。 + +## 实机验收顺序 + +- 首先确认原有触摸、AI 键、BLE、显示旋转和 OTA 确认界面工作正常。 +- 检查相机能连续采集，记录 `gesture_input` 周期与内存日志；缺相机时面板应继续工作。 +- 在每个屏幕朝向检查左右语义。默认相机安装修正为 180°、镜像开启；若方向或手形识别不正确，使用 `menuconfig → Camera gestures` 校准后重建，不能把主机坐标转换测试当作硬件方向验证。 +- 分别在三个用量页进入时钟，再挥手，确认回到原页且不继续翻页。保持 OK、自然松手和单帧漏检都不应反复切换。 +- 测试设置/配对/OTA 期间持续挥手，退出后不能回放旧动作；随后移开手再做新动作应恢复。 +- 记录实际距离、照明和方向；每朝向每方向至少 20 次，并在打字、拿杯等场景观察至少 30 分钟误触。连续运行及反复启停，检查内存是否稳定。 +- 同亮度、同 BLE 负载比较手势关闭、正常页启用、屏保启用时的电流和界面响应。尚未测得的数据不得标为通过。 + +识别距离、方向标定、模型延迟、功耗、实际摄像头热插拔和 A/B 回滚效果均为硬件验收项；初版只提供关闭/开启重试，不承诺热插拔自动恢复。 + +## 本次软件验证 + +- ESP-IDF v6.1：手势开启/关闭固件均构建成功；Mosaico logic 和共享 panel_state 测试镜像构建成功；受共享屏保接口影响的 AMOLED 2.16 固件构建成功。 +- 主机实际运行 31 个手势/屏保 Unity 测试，ASan/UBSan 通过；测试镜像未在 MCU 上运行。 +- Bridge：620 项测试通过，7 项跳过，包含新 7 MiB 发布清单容量和第三方许可检查。 +- 未进行实机烧录。ESP-DL 内部模型分配包含断言；预检查不能替代内存压力与启停实测。 diff --git a/firmware/components/esp_desktop_buddy_folder_push/Kconfig b/firmware/components/esp_desktop_buddy_folder_push/Kconfig index 51a2a2b..24a8980 100644 --- a/firmware/components/esp_desktop_buddy_folder_push/Kconfig +++ b/firmware/components/esp_desktop_buddy_folder_push/Kconfig @@ -23,7 +23,7 @@ config ESP_DESKTOP_BUDDY_FOLDER_PUSH_MAX_DECODED_CHUNK config ESP_DESKTOP_BUDDY_FOLDER_PUSH_MAX_TRANSFER_BYTES int "Max transfer size bytes" - range 0 4194816 + range 0 7340544 default 4194816 help Maximum total decoded payload bytes accepted by a single `char_begin` diff --git a/firmware/components/esp_desktop_buddy_folder_push/UPSTREAM.md b/firmware/components/esp_desktop_buddy_folder_push/UPSTREAM.md index 00e9541..26eb94e 100644 --- a/firmware/components/esp_desktop_buddy_folder_push/UPSTREAM.md +++ b/firmware/components/esp_desktop_buddy_folder_push/UPSTREAM.md @@ -8,3 +8,7 @@ This component is a vendored source snapshot from: - License: Apache-2.0 (see the SPDX headers in the source files) The upstream component manifest uses a sibling `path` dependency on `../esp_desktop_buddy`. ESP-IDF Component Manager does not include that sibling when it downloads a Git subdirectory component, so the pinned source is kept locally while `esp_desktop_buddy` itself remains a managed Git dependency. + +The transfer-size Kconfig range permits up to 7 MiB plus 512 manifest bytes for +the Mosaico gesture firmware. The default remains 4 MiB plus 512 bytes; each +target selects its limit, and the OTA backend independently checks slot capacity. diff --git a/firmware/components/usage_panel_state/include/usage_panel_state/screensaver_state.hpp b/firmware/components/usage_panel_state/include/usage_panel_state/screensaver_state.hpp index c683598..89750de 100644 --- a/firmware/components/usage_panel_state/include/usage_panel_state/screensaver_state.hpp +++ b/firmware/components/usage_panel_state/include/usage_panel_state/screensaver_state.hpp @@ -35,6 +35,8 @@ class ScreensaverController { uint64_t idle_delay_ms = kDefaultIdleDelayMs); void reset(uint64_t now_ms, Page page); + /** Enters the clock once; repeated requests preserve the original restore page. */ + void enter_clock(uint64_t now_ms, Page current_page); /** Zero disables automatic entry; explicit toggles remain available. */ void set_idle_delay(uint64_t delay_ms) { idle_delay_ms_ = delay_ms; } void note_touch_down(uint64_t now_ms); diff --git a/firmware/components/usage_panel_state/screensaver_state.cpp b/firmware/components/usage_panel_state/screensaver_state.cpp index 9eb120e..7009b47 100644 --- a/firmware/components/usage_panel_state/screensaver_state.cpp +++ b/firmware/components/usage_panel_state/screensaver_state.cpp @@ -56,6 +56,11 @@ void ScreensaverController::enter(uint64_t now_ms, Page current_page) ignore_wake_until_ms_ = 0; } +void ScreensaverController::enter_clock(uint64_t now_ms, Page current_page) +{ + if (!active_) enter(now_ms, current_page); +} + void ScreensaverController::exit(uint64_t now_ms) { active_ = false; diff --git a/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md b/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md new file mode 100644 index 0000000..c161103 --- /dev/null +++ b/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md @@ -0,0 +1,47 @@ +# Camera and gesture dependencies + +The target uses registry components pinned by `dependencies.lock`: + +| Component | Version | Role | +| --- | --- | --- | +| espressif/esp_video | 2.5.0 | DVP/V4L2 capture and buffer lifetime | +| espressif/esp_cam_sensor | 2.6.0 | OV3640 sensor configuration | +| espressif/esp-dl | 3.3.11 | S31 inference and preprocessing | +| espressif/hand_detect | 0.2.0 | ESPDet-Pico 224×224 hand detection | +| espressif/hand_gesture_recognition | 0.2.0 | MobileNetV2 128×128 classification; `ok` command | + +The published model components explicitly select their `models/p4` resources +for `esp32s31`. Both models are embedded in the application so A/B update and +rollback keep model weights and code together. Only RGB888 conversions are +enabled in ESP-DL; camera UYVY conversion and geometric mapping live in +`main/gesture_image.cpp`. No managed component is modified. + +Left-slot wiring and USB Serial/JTAG pad arbitration in +`main/gesture_camera.cpp` are adapted from `components/subboard_support/subboard.c` +in the esp-mosaico-claw source snapshot +`903fe11ec3ca4a8881afef104e53a8da040896ac` (Apache-2.0, Espressif Systems). +The pin assignments were read from the committed snapshot, not its working-tree +edits. This integration does not import the module manager, board manager, Lua, +or the esp-claw video wrapper. Preserve the copyright notice in the adapted file. + +The camera uses its onboard oscillator and the existing BSP subboard I2C bus. +GPIO14 is D4 during capture; GPIO33 is D2 and cannot concurrently serve USB +Serial/JTAG. GPIO34 is the active-low illuminator and remains high. UART logging +is retained. Start/stop and frame ownership belong to the vision worker; the +application task exchanges context and bounded events only. + +Camera mounting rotation and mirror defaults remain subject to physical +calibration. `CONFIG_MOSAICO_CAMERA_ROTATION` applies the mounting correction; +the inverse of the actual applied display rotation follows it. A context change +discards any unfinished gesture. The default correction is 180 degrees, with +horizontal mirroring enabled. + +Package copyright and license sources are recorded in +[THIRD_PARTY_LICENSES.md](../../../THIRD_PARTY_LICENSES.md). + +## Embedded model digests + +SHA-256 of the selected published S31/P4 model files: + +- `espdet_pico_224_224_hand.espdl` (576944 bytes): `a8d1857bab33bfd23bc72e1543a156ef33645b47b9ac130776aacb6a215439ea` +- `mobilenetv2_0_5_128_128_gesture.espdl` (935136 bytes): `fc4c4e0b4ba52e651dd802d5e0dc738f9eac899b107a9ce4638635e746be3c5d` diff --git a/firmware/targets/esp_mosaico/README.md b/firmware/targets/esp_mosaico/README.md index a123257..d80c543 100644 --- a/firmware/targets/esp_mosaico/README.md +++ b/firmware/targets/esp_mosaico/README.md @@ -25,7 +25,7 @@ eim run "idf.py -C firmware/targets/esp_mosaico --preview set-target esp32s31" v eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 ``` -构建产物为 `build/mosaico_usage_panel.bin`，两个 app 分区均为 4 MiB，Folder Push 总传输限额为 4 MiB + 512 字节，额外空间用于 manifest。 +构建产物为 `build/mosaico_usage_panel.bin`，两个 app 分区均为 7 MiB，Folder Push 总传输限额为 7 MiB + 512 字节，额外空间用于 manifest。 ## 当前实现 @@ -36,6 +36,12 @@ eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 - `usage_core` / `usage_protocol` / `usage_ble` / `usage_ota` 与其他目标共用，BLE 广播前缀 `QF-Mosaico-`，状态身份 `ESP-Mosaico`，型号标识 `esp_mosaico`； - 触控标签与左右滑动切页之外，板载 AI 键单击可唤醒并切换到下一页。 +### 本地手势输入 + +左槽 OV3640 CameraBoard 可选开启本地手势：左挥下一页、右挥上一页，时钟中首挥只恢复原页；👌 OK 稳定保持约 500 ms 进入时钟。设置中的 `HAND GESTURES` 默认关闭，需保存后启用。图像仅在本机处理。 + +模型与代码内嵌同一应用，首次从旧 4 MiB 分区升级必须有线迁移。保留设置时使用分段烧录，不能用填充后的完整镜像替代。依赖来源见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)，安装、恢复与实机验证见[手势验收指南](../../../docs/validation/mosaico-gestures.md)。当前仅通过软件测试和构建；相机方向、识别率、性能、功耗和提示布局仍待设备验收。 + ### 显示刷新 默认保留 CO5300 TE 输出，使用局部刷新和单个 PSRAM 绘图缓冲。BSP 在每轮 @@ -66,7 +72,7 @@ TE 信号缺失时最多等待 25 ms 后继续刷新，并在超时、恢复状 eim run "idf.py -C firmware/targets/esp_mosaico/test_apps/logic build" v6.1 ``` -只覆盖本目标独有的 `local_clock` 和 `orientation`。共享面板行为由 `firmware/test_apps/panel_state` 覆盖。没有实机时该命令只证明测试镜像编译通过。 +覆盖本目标独有的 `local_clock`、`orientation`、手势轨迹/动作路由和图像坐标转换。共享面板行为由 `firmware/test_apps/panel_state` 覆盖。没有实机时该命令只证明测试镜像编译通过。 ## 后续步骤 diff --git a/firmware/targets/esp_mosaico/dependencies.lock b/firmware/targets/esp_mosaico/dependencies.lock index 80c305b..570952e 100644 --- a/firmware/targets/esp_mosaico/dependencies.lock +++ b/firmware/targets/esp_mosaico/dependencies.lock @@ -74,12 +74,76 @@ dependencies: type: service version: 0.5.3 espressif/dhara: - component_hash: 88968cff2e88853a29366ad556e9c27596d18e15028b6e9f3cd627665f914440 + component_hash: 46b686fd0599aed21fee222c249744f05eed28d8e62446bcc83b6b50bcc390e0 dependencies: [] source: registry_url: https://components.espressif.com type: service - version: 1.0.0 + version: 1.1.0 + espressif/dl_fft: + component_hash: 3e8f7d34e04bfa6333f289a6d126745bebe0cee9311b81a6a5d106047f63ef65 + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + version: 0.7.0 + espressif/esp-dl: + component_hash: 233ee8d37dfedeb637fc74713b87bd861ac530cfbcc4c63c57c2e51064cdda31 + dependencies: + - name: espressif/dl_fft + registry_url: https://components.espressif.com + require: private + version: '>=0.6.0' + - name: espressif/esp_new_jpeg + registry_url: https://components.espressif.com + require: private + version: ^1 + - name: idf + require: private + version: '*' + source: + registry_url: https://components.espressif.com + type: service + targets: + - esp32 + - esp32c2 + - esp32c3 + - esp32c5 + - esp32c6 + - esp32p4 + - esp32s2 + - esp32s3 + - esp32s31 + version: 3.3.11 + espressif/esp_cam_sensor: + component_hash: 0dd6634a71cc8756f0e46c961476304a23a82db4f4c4deb8aecfa30161d2185c + dependencies: + - name: espressif/cmake_utilities + registry_url: https://components.espressif.com + require: private + version: 0.* + - name: espressif/esp_sccb_intf + registry_url: https://components.espressif.com + require: private + version: '>=0.0.5' + - name: idf + require: private + version: '>=5.3' + source: + registry_url: https://components.espressif.com + type: service + targets: + - esp32p4 + - esp32s3 + - esp32c3 + - esp32c5 + - esp32c6 + - esp32c61 + - esp32s31 + version: 2.6.0 espressif/esp_codec_dev: component_hash: 4779f31a3ba3f9b38aee1afe6ccb9ba8f6108dc3a6e77d39d7d714e9a215371b dependencies: @@ -259,6 +323,62 @@ dependencies: - esp32c61 - esp32h4 version: 1.0.2 + espressif/esp_sccb_intf: + component_hash: c071b189e49f40940722aea01a5489f873385cc39cd5b14012341135b85d1a9d + dependencies: + - name: idf + require: private + version: '>=5.3' + source: + registry_url: https://components.espressif.com + type: service + version: 0.0.8 + espressif/esp_video: + component_hash: 5694b16c12b8bb26b56ad3f999ad10119f1314d64b2d1c518bc7dfed902df9e1 + dependencies: + - name: espressif/cmake_utilities + registry_url: https://components.espressif.com + require: private + version: 0.* + - name: espressif/esp_cam_sensor + registry_url: https://components.espressif.com + require: private + version: 2.6.* + - matches: + - if: $CONFIG{ESP_VIDEO_USE_CUSTOMIZED_ESP_H264_VERSION} == False + name: espressif/esp_h264 + registry_url: https://components.espressif.com + require: private + rules: + - if: target in [esp32p4] + version: 1.4.* + - name: espressif/esp_ipa + registry_url: https://components.espressif.com + require: private + rules: + - if: target in [esp32p4] + version: 2.4.* + - name: idf + require: private + version: '>=5.4' + - name: espressif/usb_host_uvc + registry_url: https://components.espressif.com + require: private + rules: + - if: target in [esp32p4, esp32s3, esp32s31] + version: 2.5.* + source: + registry_url: https://components.espressif.com/ + type: service + targets: + - esp32p4 + - esp32s3 + - esp32c3 + - esp32c5 + - esp32c6 + - esp32c61 + - esp32s31 + version: 2.5.0 espressif/freetype: component_hash: aa9f7657a69b2b1c1f49044017fc424081d4e0fe6e97834cb43883548d44e3ca dependencies: @@ -269,6 +389,28 @@ dependencies: registry_url: https://components.espressif.com type: service version: 2.14.3~1 + espressif/hand_detect: + component_hash: 82b537405e093703864bd3ec7af31c3bfbc3d76cff4cf0d9e618ab7889af482c + dependencies: + - name: espressif/esp-dl + registry_url: https://components.espressif.com + require: private + version: ~3.3.0 + source: + registry_url: https://components.espressif.com/ + type: service + version: 0.2.0 + espressif/hand_gesture_recognition: + component_hash: 99389204096e649fe10e5a28e4104e814975f8a27a046b6b3c52dfc5ede1acc4 + dependencies: + - name: espressif/hand_detect + registry_url: https://components.espressif.com + require: private + version: ~0.2.0 + source: + registry_url: https://components.espressif.com/ + type: service + version: 0.2.0 espressif/i2c_bus: component_hash: 483112c1619c54c38d0b734fb25f1678733267858a77b7af2f48cc7db6dbbd84 dependencies: @@ -325,6 +467,47 @@ dependencies: registry_url: https://components.espressif.com/ type: service version: 1.4.4 + espressif/usb: + component_hash: 6c2cff118141dbbca181d0d27d71801fdddfc78f7ac03ae43f6e5a8cbd89cb99 + dependencies: + - name: idf + require: private + version: '>=5.5.3' + source: + registry_url: https://components.espressif.com + type: service + targets: + - esp32s2 + - esp32s3 + - esp32s31 + - esp32p4 + - esp32h4 + - linux + version: 1.5.0 + espressif/usb_host_uvc: + component_hash: 3dde9c89980212de1e394a29d3fe782a2988a08fd2a16af34eee6f642aab6ebe + dependencies: + - name: idf + require: private + version: '>=5.0' + - name: espressif/usb + registry_url: https://components.espressif.com + require: public + rules: + - if: idf_version >=6.0 + - if: target not in ["linux"] + version: ^1.0.0 + source: + registry_url: https://components.espressif.com + type: service + targets: + - esp32s2 + - esp32s3 + - esp32p4 + - esp32h4 + - esp32s31 + - linux + version: 2.5.1 espressif/zlib: component_hash: 9083cada65f1c9d298ef92266232d071c6f2f335164e2799554b3eae73aedc55 dependencies: @@ -387,11 +570,14 @@ direct_dependencies: - espressif/esp_lcd_co5300 - espressif/esp_lcd_touch_cst9220 - espressif/esp_lvgl_adapter +- espressif/esp_video +- espressif/hand_detect +- espressif/hand_gesture_recognition - espressif/spi_nand_flash - espressif2022/bmi270 - idf - lvgl/lvgl - tangyumei3535/bmm150_sensorapi -manifest_hash: 381e620403dd6ec5c85708e211ed42ac6af8ae3c1200c0c5392c28805b22019b +manifest_hash: e9597de848f4dedfd77d40c2df27a3229221ce044985973fb9e4d7ee9873d515 target: esp32s31 version: 3.0.0 diff --git a/firmware/targets/esp_mosaico/main/CMakeLists.txt b/firmware/targets/esp_mosaico/main/CMakeLists.txt index 7e959f5..ae8aa2b 100644 --- a/firmware/targets/esp_mosaico/main/CMakeLists.txt +++ b/firmware/targets/esp_mosaico/main/CMakeLists.txt @@ -5,6 +5,9 @@ idf_component_register( "local_clock.cpp" "orientation.cpp" "power_monitor.cpp" + "panel_settings.cpp" + "gesture_input.cpp" + "gesture_tracker.cpp" INCLUDE_DIRS "." REQUIRES nvs_flash @@ -17,3 +20,7 @@ idf_component_register( ) target_compile_features(${COMPONENT_LIB} PUBLIC cxx_std_17) + +if(CONFIG_MOSAICO_GESTURE_INPUT) + target_sources(${COMPONENT_LIB} PRIVATE "gesture_camera.cpp" "gesture_image.cpp") +endif() diff --git a/firmware/targets/esp_mosaico/main/Kconfig.projbuild b/firmware/targets/esp_mosaico/main/Kconfig.projbuild index f09a69f..4763d01 100644 --- a/firmware/targets/esp_mosaico/main/Kconfig.projbuild +++ b/firmware/targets/esp_mosaico/main/Kconfig.projbuild @@ -88,6 +88,34 @@ config MOSAICO_USAGE_PANEL_TILT_WAKE_DEGREES endmenu +menu "Camera gestures" + +config MOSAICO_GESTURE_INPUT + bool "Build local CameraBoard gesture input" + default y + help + Include OV3640 capture and ESP-DL models. Runtime capture remains off + until enabled and saved in the display settings. The camera occupies + the left slot and GPIO33, so USB Serial/JTAG cannot be used concurrently. + +config MOSAICO_CAMERA_ROTATION + int "Camera mounting correction (0, 90, 180 or 270 degrees clockwise)" + range 0 270 + default 180 + depends on MOSAICO_GESTURE_INPUT + help + Rotates the raw image into panel rotation-0 coordinates. The current + display rotation is then inverted. Calibrate this on the physical board. + +config MOSAICO_CAMERA_MIRROR + bool "Mirror the camera horizontally before rotation" + default y + depends on MOSAICO_GESTURE_INPUT + help + Mirror the front-facing camera into the user's screen coordinates. + +endmenu + menu "Bluetooth" config MOSAICO_USAGE_PANEL_BLE_ADVERTISING_PREFIX diff --git a/firmware/targets/esp_mosaico/main/app_main.cpp b/firmware/targets/esp_mosaico/main/app_main.cpp index 64f8cd4..a7b55d6 100644 --- a/firmware/targets/esp_mosaico/main/app_main.cpp +++ b/firmware/targets/esp_mosaico/main/app_main.cpp @@ -4,6 +4,8 @@ #include "bsp/esp_mosaico.h" #include "display_ui.hpp" #include "local_clock.hpp" +#include "gesture_input.hpp" +#include "panel_settings.hpp" #include "usage_panel_state/panel_presentation.hpp" #include "power_monitor.hpp" #include "usage_panel_state/render_schedule.hpp" @@ -80,6 +82,7 @@ OtaSession ota(ota_backend); AtomicPowerSource ota_power_source; BleService ble; DisplayUi ui; +GestureInput gestures; LinkUpdate ble_link{}; std::atomic status_page{Page::Overview}; std::atomic ai_button_pressed{false}; @@ -191,11 +194,12 @@ extern "C" void app_main(void) } const bool ai_button_ready = init_ai_button(); - DisplaySettings saved_settings = load_display_settings({ + PanelSettings saved_settings = load_panel_settings({ CONFIG_MOSAICO_USAGE_PANEL_DISPLAY_BRIGHTNESS_PERCENT, CONFIG_MOSAICO_USAGE_PANEL_SCREENSAVER_TIMEOUT_SECONDS}); - DisplaySettings settings = saved_settings; + PanelSettings settings = saved_settings; bool settings_open = false; + if (!gestures.begin()) ESP_LOGW(TAG, "gesture worker unavailable"); ui.apply_settings(settings, false); ScreenRotation desired_rotation = initial_rotation; @@ -259,6 +263,7 @@ extern "C" void app_main(void) dirty = true; } if (ui.take_touch_down()) { + gestures.invalidate(); const bool was_asleep = screensaver.view(now_ms).active; screensaver.note_touch_down(now_ms); if (was_asleep) { @@ -269,6 +274,7 @@ extern "C" void app_main(void) } if (ai_button_ready && ai_button_pressed.exchange(false, std::memory_order_relaxed)) { + gestures.invalidate(); const bool was_asleep = screensaver.view(now_ms).active; screensaver.note_touch_down(now_ms); if (was_asleep) { @@ -294,9 +300,14 @@ extern "C" void app_main(void) screensaver.reset(now_ms, page); dirty = true; } + if (ui.take_gesture_toggle() && settings_open && !settings_blocked) { + settings.gestures = !settings.gestures; + ui.apply_settings(settings, true, false, settings_differ(settings, saved_settings)); + dirty = true; + } if (action != SettingsAction::None && !settings_blocked) { bool save_error = false; - const DisplaySettings before = settings; + const PanelSettings before = settings; const bool was_open = settings_open; if (action == SettingsAction::Open) settings_open = true; else if (settings_open) { @@ -307,7 +318,7 @@ extern "C" void app_main(void) else if (action >= SettingsAction::TimeoutOff && action <= SettingsAction::Timeout30) select_clock_timeout(action, settings.clock_timeout_seconds); else if (action == SettingsAction::Save) { - const esp_err_t error = save_display_settings(settings); + const esp_err_t error = save_panel_settings(settings); save_error = error != ESP_OK; if (!save_error) { saved_settings = settings; settings_open = false; } else ESP_LOGW(TAG, "display settings save failed: %s", esp_err_to_name(error)); @@ -317,8 +328,7 @@ extern "C" void app_main(void) } } if (!ui.apply_settings(settings, settings_open, save_error, - settings.brightness != saved_settings.brightness || - settings.clock_timeout_seconds != saved_settings.clock_timeout_seconds)) { + settings_differ(settings, saved_settings))) { settings = before; settings_open = was_open; ESP_LOGW(TAG, "display settings preview failed"); @@ -329,6 +339,7 @@ extern "C" void app_main(void) } const Page requested = ui.requested_page(); if (requested != page) { + gestures.invalidate(); page = requested; status_page.store(page, std::memory_order_relaxed); dirty = true; @@ -339,8 +350,10 @@ extern "C" void app_main(void) if (event.type == AppEventType::Usage) { model.apply(event.usage, esp_timer_get_time() / 1000); } else if (event.type == AppEventType::Link) { + gestures.invalidate(); ble_link = event.link; } else if (event.type == AppEventType::ScreenPage) { + gestures.invalidate(); if (ota_view.phase == OtaPhase::Idle && !ble_link.has_passkey) { if (screensaver.view(now_ms).active) page = screensaver.restore_page(); page = page_after_swipe(page, event.previous_page @@ -353,6 +366,7 @@ extern "C" void app_main(void) status_page.store(page, std::memory_order_relaxed); } } else if (event.type == AppEventType::ScreenToggle) { + gestures.invalidate(); settings = saved_settings; settings_open = false; ui.apply_settings(settings, false); @@ -438,6 +452,7 @@ extern "C" void app_main(void) woke = tilt_detector.update(accel_x, accel_y, accel_z) || woke; #endif if (woke) { + gestures.invalidate(); screensaver.note_shake(now_ms); dirty = true; } @@ -455,6 +470,32 @@ extern "C" void app_main(void) screensaver.update( now_ms, page, ble_link.has_passkey, ota_view.phase != OtaPhase::Idle || show_ota_error || settings_open); + const bool gesture_allowed = !settings_open && !ble_link.has_passkey && + ota_view.phase == OtaPhase::Idle && !show_ota_error && desired_rotation == applied_rotation; + gestures.configure(saved_settings.gestures, gesture_allowed, + screensaver.view(now_ms).active, page, applied_rotation); + GestureEvent gesture{}; + if (gestures.take(gesture, esp_timer_get_time() / 1000)) { + const GestureEffect effect = route_gesture(gesture.action, gesture_allowed, + screensaver.view(now_ms).active); + if (effect == GestureEffect::Wake) { + screensaver.note_touch_down(now_ms); + if (!screensaver.view(now_ms).active) page = screensaver.restore_page(); + } else if (effect == GestureEffect::Clock) { + screensaver.enter_clock(now_ms, page); + } else if (effect == GestureEffect::Next || effect == GestureEffect::Previous) { + page = page_after_swipe(page, effect == GestureEffect::Next + ? SwipeDirection::Next : SwipeDirection::Previous); + screensaver.reset(now_ms, page); + } + if (effect != GestureEffect::None) { + ui.request_page(page); + status_page.store(page, std::memory_order_relaxed); + gestures.invalidate(); + ESP_LOGI(TAG, "gesture action=%u effect=%u", unsigned(gesture.action), unsigned(effect)); + dirty = true; + } + } const ScreensaverView screensaver_view = screensaver.view(now_ms); const PanelPresentation presentation{ .clock = format_clock_view( @@ -488,6 +529,19 @@ extern "C" void app_main(void) has_last_render_key = true; dirty = false; } + const auto gesture_status = gestures.status(); + const auto gesture_progress = gestures.progress(); + const bool gesture_feedback_visible = saved_settings.gestures && gesture_allowed && !screensaver_view.active; + static GestureStatus shown_gesture_status = GestureStatus::Off; + static uint8_t shown_gesture_progress = 0; + static bool shown_gesture_feedback = false; + if (shown_gesture_status != gesture_status || shown_gesture_progress != gesture_progress || + shown_gesture_feedback != gesture_feedback_visible) { + ui.show_gesture_status(gesture_status, gesture_progress, gesture_feedback_visible); + shown_gesture_status = gesture_status; + shown_gesture_progress = gesture_progress; + shown_gesture_feedback = gesture_feedback_visible; + } last_checked_second = second; ble.verify_boot(); diff --git a/firmware/targets/esp_mosaico/main/display_ui.cpp b/firmware/targets/esp_mosaico/main/display_ui.cpp index c08b15f..7c3e3be 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.cpp +++ b/firmware/targets/esp_mosaico/main/display_ui.cpp @@ -769,20 +769,20 @@ void DisplayUi::build_settings(lv_obj_t* screen) lv_obj_set_style_bg_color(settings_panel_, lv_color_hex(COLOR_BG), 0); lv_obj_set_style_bg_opa(settings_panel_, LV_OPA_COVER, 0); lv_obj_remove_flag(settings_panel_, LV_OBJ_FLAG_SCROLLABLE); - auto* title = make_label(settings_panel_, "BRIGHTNESS", MARGIN, 24, + auto* title = make_label(settings_panel_, "BRIGHTNESS", MARGIN, 16, &lv_font_montserrat_16, COLOR_MUTED); lv_obj_set_style_text_letter_space(title, 1, 0); - settings_brightness_ = make_label(settings_panel_, "", MARGIN, 58, &lv_font_montserrat_48); + settings_brightness_ = make_label(settings_panel_, "", MARGIN, 42, &lv_font_montserrat_48); lv_obj_t* divider = lv_obj_create(settings_panel_); lv_obj_remove_style_all(divider); - lv_obj_set_pos(divider, MARGIN, 158); + lv_obj_set_pos(divider, MARGIN, 128); lv_obj_set_size(divider, CARD_W, 1); lv_obj_set_style_bg_color(divider, lv_color_hex(COLOR_LINE), 0); lv_obj_set_style_bg_opa(divider, LV_OPA_COVER, 0); - title = make_label(settings_panel_, "AUTO CLOCK", MARGIN, 180, + title = make_label(settings_panel_, "AUTO CLOCK", MARGIN, 145, &lv_font_montserrat_16, COLOR_MUTED); lv_obj_set_style_text_letter_space(title, 1, 0); - settings_timeout_ = make_label(settings_panel_, "AFTER IDLE / MIN", MARGIN, 207, + settings_timeout_ = make_label(settings_panel_, "AFTER IDLE / MIN", MARGIN, 170, &lv_font_montserrat_14, COLOR_MUTED); settings_hint_ = make_label(settings_panel_, "", MARGIN, 322, &montserrat_semibold_13, COLOR_TEXT); @@ -791,13 +791,13 @@ void DisplayUi::build_settings(lv_obj_t* screen) SettingsAction::TimeoutOff, SettingsAction::Timeout1, SettingsAction::Timeout5, SettingsAction::Timeout10, SettingsAction::Timeout30, SettingsAction::Save, SettingsAction::Cancel}; const int x[] = {288, 376, 24, 112, 200, 288, 376, 250, MARGIN}; - const int y[] = {58, 58, 244, 244, 244, 244, 244, 349, 349}; + const int y[] = {38, 38, 200, 200, 200, 200, 200, 349, 349}; for (size_t i = 0; i < 9; ++i) { lv_obj_t* button = lv_button_create(settings_panel_); settings_buttons_[i] = button; lv_obj_remove_style_all(button); lv_obj_set_pos(button, x[i], y[i]); - lv_obj_set_size(button, i < 7 ? 80 : 206, i < 2 ? 80 : 64); + lv_obj_set_size(button, i < 7 ? 80 : 206, i < 2 ? 72 : (i < 7 ? 52 : 64)); style_panel(button); lv_obj_set_style_text_color(button, lv_color_hex(COLOR_MUTED), 0); lv_obj_set_style_text_font(button, i < 2 ? &lv_font_montserrat_30 : &lv_font_montserrat_20, 0); @@ -816,10 +816,34 @@ void DisplayUi::build_settings(lv_obj_t* screen) lv_label_set_text(label, labels[i]); lv_obj_center(label); } +#if CONFIG_MOSAICO_GESTURE_INPUT + gesture_toggle_button_ = lv_button_create(settings_panel_); + lv_obj_remove_style_all(gesture_toggle_button_); + lv_obj_set_pos(gesture_toggle_button_, MARGIN, 270); + lv_obj_set_size(gesture_toggle_button_, CARD_W, 44); + style_panel(gesture_toggle_button_); + lv_obj_set_style_bg_color(gesture_toggle_button_, lv_color_hex(COLOR_TRACK), LV_STATE_PRESSED); + gesture_toggle_label_ = make_label(gesture_toggle_button_, "HAND GESTURES / OFF", 12, 12, + &montserrat_semibold_13, COLOR_TEXT); + lv_obj_add_event_cb(gesture_toggle_button_, gesture_toggle_event, LV_EVENT_CLICKED, this); + gesture_status_label_ = make_label(screen, "", MARGIN, HEADER_H + 8, + &montserrat_semibold_13, COLOR_MUTED); + lv_obj_add_flag(gesture_status_label_, LV_OBJ_FLAG_HIDDEN); + gesture_progress_ = lv_bar_create(screen); + lv_obj_remove_style_all(gesture_progress_); + lv_obj_set_pos(gesture_progress_, MARGIN, HEADER_H - 4); + lv_obj_set_size(gesture_progress_, CARD_W, 3); + lv_bar_set_range(gesture_progress_, 0, 100); + lv_obj_set_style_bg_color(gesture_progress_, lv_color_hex(COLOR_TRACK), LV_PART_MAIN); + lv_obj_set_style_bg_opa(gesture_progress_, LV_OPA_COVER, LV_PART_MAIN); + lv_obj_set_style_bg_color(gesture_progress_, lv_color_hex(COLOR_RESET), LV_PART_INDICATOR); + lv_obj_set_style_bg_opa(gesture_progress_, LV_OPA_COVER, LV_PART_INDICATOR); + lv_obj_add_flag(gesture_progress_, LV_OBJ_FLAG_HIDDEN); +#endif lv_obj_add_flag(settings_panel_, LV_OBJ_FLAG_HIDDEN); } -bool DisplayUi::apply_settings(const DisplaySettings& settings, bool visible, bool save_error, bool modified) +bool DisplayUi::apply_settings(const PanelSettings& settings, bool visible, bool save_error, bool modified) { DisplayLock lock; if (!lock) return false; @@ -830,6 +854,8 @@ bool DisplayUi::apply_settings(const DisplaySettings& settings, bool visible, bo return false; brightness_percent_ = settings.brightness; } + if (gesture_toggle_label_) + lv_label_set_text(gesture_toggle_label_, settings.gestures ? "HAND GESTURES / ON" : "HAND GESTURES / OFF"); const bool was_visible = settings_visible_.exchange(visible, std::memory_order_relaxed); lv_label_set_text_fmt(settings_brightness_, "%u%%", settings.brightness); lv_obj_set_state(settings_buttons_[0], LV_STATE_DISABLED, settings.brightness <= 1); @@ -852,6 +878,35 @@ bool DisplayUi::apply_settings(const DisplaySettings& settings, bool visible, bo return true; } +void DisplayUi::gesture_toggle_event(lv_event_t* event) +{ + auto* ui = static_cast(lv_event_get_user_data(event)); + if (ui->settings_visible_.load()) ui->gesture_toggle_ = true; +} + +void DisplayUi::show_gesture_status(GestureStatus status, uint8_t progress, bool visible) +{ + if (!gesture_status_label_) return; + DisplayLock lock; + if (!lock) return; + const char* text = ""; + if (visible) { + if (status == GestureStatus::Starting) text = "CAMERA STARTING"; + else if (status == GestureStatus::CameraError) text = "CHECK LEFT CAMERA / RETRY IN SETTINGS"; + else if (status == GestureStatus::MemoryError) text = "GESTURES UNAVAILABLE / LOW MEMORY"; + else if (status == GestureStatus::Fault) text = "GESTURES UNAVAILABLE / RESTART DEVICE"; + else if (progress) text = "HOLD OK TO ENTER CLOCK"; + } + lv_label_set_text(gesture_status_label_, text); + lv_obj_set_flag(gesture_status_label_, LV_OBJ_FLAG_HIDDEN, !text[0]); + lv_obj_set_flag(gesture_progress_, LV_OBJ_FLAG_HIDDEN, !visible || !progress); + if (text[0]) lv_obj_move_foreground(gesture_status_label_); + if (visible && progress) { + lv_bar_set_value(gesture_progress_, progress, LV_ANIM_OFF); + lv_obj_move_foreground(gesture_progress_); + } +} + void DisplayUi::settings_event(lv_event_t* event) { auto* binding = static_cast(lv_event_get_user_data(event)); diff --git a/firmware/targets/esp_mosaico/main/display_ui.hpp b/firmware/targets/esp_mosaico/main/display_ui.hpp index bc0c7db..06ae0f8 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.hpp +++ b/firmware/targets/esp_mosaico/main/display_ui.hpp @@ -6,6 +6,8 @@ #include "esp_lcd_panel_io.h" #include "lvgl.h" #include "orientation.hpp" +#include "panel_settings.hpp" +#include "gesture_input.hpp" #include "usage_panel_state/display_settings.hpp" #include "usage_panel_state/page_state.hpp" #include "usage_panel_state/panel_presentation.hpp" @@ -29,7 +31,9 @@ class DisplayUi { bool begin(ScreenRotation initial_rotation); void request_page(Page page) { requested_page_.store(page, std::memory_order_relaxed); } - bool apply_settings(const DisplaySettings& settings, bool visible, bool save_error = false, bool modified = false); + bool apply_settings(const PanelSettings& settings, bool visible, bool save_error = false, bool modified = false); + bool take_gesture_toggle() { return gesture_toggle_.exchange(false); } + void show_gesture_status(GestureStatus status, uint8_t progress, bool visible); SettingsAction take_settings_action() { return settings_action_.exchange(SettingsAction::None, std::memory_order_relaxed); @@ -106,6 +110,7 @@ class DisplayUi { void build_settings(lv_obj_t* screen); static void settings_event(lv_event_t* event); + static void gesture_toggle_event(lv_event_t* event); static void long_press_event(lv_event_t* event); void build_header(lv_obj_t* screen); void build_overview(lv_obj_t* screen); @@ -140,6 +145,11 @@ class DisplayUi { lv_obj_t* settings_buttons_[9]{}; lv_obj_t* settings_timeout_ = nullptr; lv_obj_t* settings_hint_ = nullptr; + lv_obj_t* gesture_toggle_button_ = nullptr; + lv_obj_t* gesture_toggle_label_ = nullptr; + lv_obj_t* gesture_status_label_ = nullptr; + lv_obj_t* gesture_progress_ = nullptr; + std::atomic gesture_toggle_{false}; uint8_t brightness_percent_ = 50; // Serialized by the LVGL lock. std::atomic requested_page_{Page::Overview}; std::atomic ota_active_{false}; diff --git a/firmware/targets/esp_mosaico/main/gesture_camera.cpp b/firmware/targets/esp_mosaico/main/gesture_camera.cpp new file mode 100644 index 0000000..1319342 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/gesture_camera.cpp @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +// SPDX-License-Identifier: Apache-2.0 +#include "gesture_camera.hpp" +#include "gesture_image.hpp" + +#include +#include +#include +#include +#include + +#include "bsp/esp_mosaico.h" +#include "esp_log.h" +#include "esp_video_device.h" +#include "esp_video_init.h" +#include "esp_video_ioctl.h" +#include "hal/usb_serial_jtag_ll.h" +#include "linux/videodev2.h" + +namespace usage_panel::mosaico { +namespace { +constexpr char TAG[] = "gesture_camera"; +constexpr gpio_num_t kPowerDown = GPIO_NUM_48; +constexpr gpio_num_t kFlash = GPIO_NUM_34; + +bool checked_ioctl(int fd, unsigned long command, void* arg) +{ + if (ioctl(fd, command, arg) == 0) return true; + ESP_LOGW(TAG, "video ioctl 0x%lx failed: errno=%d", command, errno); + return false; +} +} + +bool GestureCamera::start() +{ + if (pins_claimed_ || fd_ >= 0) return false; + if (bsp_subboard_init() != ESP_OK) return false; + // D2 shares GPIO33 with Serial/JTAG. The UART console remains available. + restore_usb_pad_ = usb_serial_jtag_ll_phy_is_pad_enabled(); + restore_usb_clock_ = usb_serial_jtag_ll_module_is_enabled(); + restore_usb_interrupts_ = restore_usb_clock_ ? usb_serial_jtag_ll_get_intr_ena_status() : 0; + if (restore_usb_clock_) usb_serial_jtag_ll_disable_intr_mask(USB_SERIAL_JTAG_LL_INTR_MASK); + usb_serial_jtag_ll_phy_enable_pad(false); + usb_serial_jtag_ll_enable_bus_clock(false); + pins_claimed_ = true; + gpio_reset_pin(GPIO_NUM_14); + gpio_set_direction(GPIO_NUM_14, GPIO_MODE_INPUT); + // The illuminator is active-low and must never be driven by gesture capture. + gpio_set_level(kFlash, 1); + gpio_set_direction(kFlash, GPIO_MODE_OUTPUT); + + esp_video_init_dvp_config_t dvp{}; + dvp.sccb_config.init_sccb = false; + dvp.sccb_config.i2c_handle = bsp_subboard_get_i2c_bus(); + dvp.sccb_config.freq = 400000; + dvp.reset_pin = GPIO_NUM_53; + dvp.pwdn_pin = kPowerDown; + dvp.dvp_pin.data_width = CAM_CTLR_DATA_WIDTH_8; + const gpio_num_t data[] = {GPIO_NUM_16, GPIO_NUM_15, GPIO_NUM_33, GPIO_NUM_4, + GPIO_NUM_14, GPIO_NUM_12, GPIO_NUM_18, GPIO_NUM_13}; + for (unsigned i = 0; i < 8; ++i) dvp.dvp_pin.data_io[i] = data[i]; + dvp.dvp_pin.vsync_io = GPIO_NUM_55; + dvp.dvp_pin.de_io = GPIO_NUM_19; + dvp.dvp_pin.pclk_io = GPIO_NUM_17; + dvp.dvp_pin.xclk_io = GPIO_NUM_NC; + dvp.xclk_freq = 0; // CameraBoard supplies its own 24 MHz oscillator. + esp_video_init_config_t config{}; + config.dvp = &dvp; + const esp_err_t error = esp_video_init_with_flags(&config, ESP_VIDEO_INIT_FLAGS_DVP); + if (error != ESP_OK) { + ESP_LOGW(TAG, "left-slot OV3640 unavailable: %s", esp_err_to_name(error)); + return false; + } + video_initialized_ = true; + fd_ = open(ESP_VIDEO_DVP_DEVICE_NAME, O_RDWR); + if (fd_ < 0) return false; + v4l2_format format{}; + format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + format.fmt.pix.width = 640; + format.fmt.pix.height = 480; + format.fmt.pix.pixelformat = V4L2_PIX_FMT_UYVY; + if (!checked_ioctl(fd_, VIDIOC_S_FMT, &format)) return false; + if (format.fmt.pix.pixelformat != V4L2_PIX_FMT_UYVY || + format.fmt.pix.width != 640 || format.fmt.pix.height != 480) return false; + width_ = format.fmt.pix.width; + height_ = format.fmt.pix.height; + stride_ = format.fmt.pix.bytesperline; + if (!stride_) stride_ = width_ * 2; + timeval timeout{0, 200000}; + if (!checked_ioctl(fd_, VIDIOC_S_DQBUF_TIMEOUT, &timeout)) return false; + v4l2_requestbuffers request{}; + request.count = 2; + request.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + request.memory = V4L2_MEMORY_MMAP; + if (!checked_ioctl(fd_, VIDIOC_REQBUFS, &request) || request.count != 2) return false; + for (unsigned i = 0; i < 2; ++i) { + v4l2_buffer buffer{}; + buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buffer.memory = V4L2_MEMORY_MMAP; + buffer.index = i; + if (!checked_ioctl(fd_, VIDIOC_QUERYBUF, &buffer)) return false; + buffers_[i] = mmap(nullptr, buffer.length, PROT_READ | PROT_WRITE, + MAP_SHARED, fd_, buffer.m.offset); + if (buffers_[i] == MAP_FAILED) { buffers_[i] = nullptr; return false; } + lengths_[i] = buffer.length; + if (!checked_ioctl(fd_, VIDIOC_QBUF, &buffer)) return false; + } + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + streaming_ = checked_ioctl(fd_, VIDIOC_STREAMON, &type); + if (streaming_) ESP_LOGI(TAG, "OV3640 UYVY %ux%u stride=%u buffers=2", width_, height_, stride_); + return streaming_; +} + +bool GestureCamera::read_rgb(uint8_t* output, unsigned rotation, bool mirror) +{ + v4l2_buffer buffer{}; + buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buffer.memory = V4L2_MEMORY_MMAP; + if (!streaming_ || !checked_ioctl(fd_, VIDIOC_DQBUF, &buffer)) return false; + // At most two capture buffers exist. Prefer the newer completed frame if + // inference left both queued; never accumulate camera latency across cycles. + timeval nonblocking{}; + bool timeout_ok = checked_ioctl(fd_, VIDIOC_S_DQBUF_TIMEOUT, &nonblocking); + if (timeout_ok) { + v4l2_buffer newer{}; + newer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + newer.memory = V4L2_MEMORY_MMAP; + if (ioctl(fd_, VIDIOC_DQBUF, &newer) == 0) { + const bool returned = checked_ioctl(fd_, VIDIOC_QBUF, &buffer); + buffer = newer; + timeout_ok = returned; + } + timeval blocking{0, 200000}; + timeout_ok = checked_ioctl(fd_, VIDIOC_S_DQBUF_TIMEOUT, &blocking) && timeout_ok; + } + bool converted = false; + if (buffer.index < 2 && buffers_[buffer.index]) { + const size_t bytes = buffer.bytesused ? buffer.bytesused : lengths_[buffer.index]; + if (bytes <= lengths_[buffer.index]) + converted = prepare_gesture_image(static_cast(buffers_[buffer.index]), + bytes, width_, height_, stride_, output, rotation, mirror); + } + const bool returned = checked_ioctl(fd_, VIDIOC_QBUF, &buffer); + return converted && returned && timeout_ok; +} + +bool GestureCamera::stop() +{ + if (streaming_) { + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (!checked_ioctl(fd_, VIDIOC_STREAMOFF, &type)) return false; + streaming_ = false; + } + if (fd_ >= 0) { + for (unsigned i = 0; i < 2; ++i) { + if (buffers_[i]) { + if (munmap(buffers_[i], lengths_[i]) != 0) return false; + buffers_[i] = nullptr; + } + } + if (close(fd_) != 0) return false; + fd_ = -1; + } + if (video_initialized_) { + if (esp_video_deinit_with_flags(ESP_VIDEO_INIT_FLAGS_DVP) != ESP_OK) return false; + video_initialized_ = false; + } + if (pins_claimed_) { + gpio_set_level(kPowerDown, 1); + gpio_set_direction(kPowerDown, GPIO_MODE_OUTPUT); + gpio_set_level(kFlash, 1); + // DVP is stopped before restoring the shared pads and EEPROM address line. + gpio_reset_pin(GPIO_NUM_33); + if (restore_usb_clock_) usb_serial_jtag_ll_enable_bus_clock(true); + usb_serial_jtag_ll_phy_enable_pad(restore_usb_pad_); + if (restore_usb_clock_ && restore_usb_interrupts_) + usb_serial_jtag_ll_ena_intr_mask(restore_usb_interrupts_); + if (bsp_subboard_apply_address_select(BSP_SUBBOARD_SLOT_LEFT) != ESP_OK) return false; + pins_claimed_ = false; + } + return true; +} + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/gesture_camera.hpp b/firmware/targets/esp_mosaico/main/gesture_camera.hpp new file mode 100644 index 0000000..21b98f8 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/gesture_camera.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +namespace usage_panel::mosaico { + +/** Owns the left-slot OV3640 and its V4L2 buffers on the vision worker only. */ +class GestureCamera { +public: + bool start(); + /** Copies/converts one frame and requeues it before returning to the caller. */ + bool read_rgb(uint8_t* output, unsigned rotation, bool mirror); + /** Returns false if resources cannot be safely released; do not restart afterward. */ + bool stop(); + +private: + int fd_ = -1; + bool pins_claimed_ = false; + bool video_initialized_ = false; + bool streaming_ = false; + bool restore_usb_pad_ = false; + bool restore_usb_clock_ = false; + uint32_t restore_usb_interrupts_ = 0; + void* buffers_[2]{}; + size_t lengths_[2]{}; + unsigned width_ = 0; + unsigned height_ = 0; + unsigned stride_ = 0; +}; + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/gesture_image.cpp b/firmware/targets/esp_mosaico/main/gesture_image.cpp new file mode 100644 index 0000000..67996d4 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/gesture_image.cpp @@ -0,0 +1,44 @@ +#include "gesture_image.hpp" + +#include + +namespace usage_panel::mosaico { +namespace { +uint8_t channel(int value) +{ + return static_cast(std::clamp(value / 256, 0, 255)); +} +} + +bool prepare_gesture_image(const uint8_t* src, size_t size, unsigned width, + unsigned height, unsigned stride, uint8_t* dst, + unsigned turns, bool mirror) +{ + if (!src || !dst || width < 2 || width > 4096 || height == 0 || height > 4096 || + (width & 1) || stride < width * 2 || size < size_t(stride) * height || turns > 3) + return false; + const unsigned out_width = turns & 1 ? kGestureImageHeight : kGestureImageWidth; + for (unsigned y = 0; y < kGestureImageHeight; ++y) { + for (unsigned x = 0; x < kGestureImageWidth; ++x) { + const unsigned sx = x * width / kGestureImageWidth; + const unsigned sy = y * height / kGestureImageHeight; + const uint8_t* p = src + size_t(sy) * stride + (sx & ~1U) * 2; + const int luma = std::max(0, int(p[(sx & 1) ? 3 : 1]) - 16); + const int u = int(p[0]) - 128; + const int v = int(p[2]) - 128; + const unsigned mx = mirror ? kGestureImageWidth - 1 - x : x; + unsigned ox = mx, oy = y; + if (turns == 1) { ox = kGestureImageHeight - 1 - y; oy = mx; } + if (turns == 2) { ox = kGestureImageWidth - 1 - mx; oy = kGestureImageHeight - 1 - y; } + if (turns == 3) { ox = y; oy = kGestureImageWidth - 1 - mx; } + uint8_t* rgb = dst + (oy * out_width + ox) * 3; + // OV3640 UYVY uses the limited-range BT.601 YCbCr conversion. + rgb[0] = channel(298 * luma + 409 * v + 128); + rgb[1] = channel(298 * luma - 100 * u - 208 * v + 128); + rgb[2] = channel(298 * luma + 516 * u + 128); + } + } + return true; +} + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/gesture_image.hpp b/firmware/targets/esp_mosaico/main/gesture_image.hpp new file mode 100644 index 0000000..f403f17 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/gesture_image.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + +namespace usage_panel::mosaico { + +constexpr unsigned kGestureImageWidth = 320; +constexpr unsigned kGestureImageHeight = 240; +constexpr size_t kGestureImageBytes = kGestureImageWidth * kGestureImageHeight * 3; + +/** Downsamples packed UYVY to RGB888, mirrors horizontally, then rotates clockwise. + * Output capacity must be at least kGestureImageBytes. Odd rotations yield 240x320. + */ +bool prepare_gesture_image(const uint8_t* src, size_t size, unsigned width, + unsigned height, unsigned stride, uint8_t* dst, + unsigned quarter_turns, bool mirror); + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/gesture_input.cpp b/firmware/targets/esp_mosaico/main/gesture_input.cpp new file mode 100644 index 0000000..d25ea10 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/gesture_input.cpp @@ -0,0 +1,223 @@ +#include "gesture_input.hpp" +#include "sdkconfig.h" + +#include "esp_log.h" +#include "esp_timer.h" +#include "freertos/task.h" + +#if CONFIG_MOSAICO_GESTURE_INPUT +#include +#include +#include +#include "esp_heap_caps.h" +#include "gesture_camera.hpp" +#include "gesture_image.hpp" +#include "hand_detect.hpp" +#include "hand_gesture_recognition.hpp" +#endif + +namespace usage_panel::mosaico { +#if CONFIG_MOSAICO_GESTURE_INPUT +static_assert(CONFIG_MOSAICO_CAMERA_ROTATION % 90 == 0, "Camera rotation must be a multiple of 90 degrees"); +#endif +namespace { +constexpr uint32_t kEnabled = 1; +constexpr uint32_t kAllowed = 2; +constexpr uint32_t kAsleep = 4; +constexpr uint32_t kContextMask = 0x7f; +constexpr uint32_t kGeneration = 0x80; +constexpr char TAG[] = "gesture_input"; +} + +bool GestureInput::begin() +{ +#if CONFIG_MOSAICO_GESTURE_INPUT + queue_ = xQueueCreate(1, sizeof(GestureEvent)); + if (!queue_) { status_ = GestureStatus::MemoryError; return false; } + if (xTaskCreate(task_entry, "hand_input", 8192, this, 3, nullptr) != pdPASS) { + vQueueDelete(queue_); + queue_ = nullptr; + status_ = GestureStatus::MemoryError; + return false; + } +#endif + return true; +} + +void GestureInput::configure(bool enabled, bool allowed, bool asleep, Page page, ScreenRotation rotation) +{ + const uint32_t flags = (enabled ? kEnabled : 0) | (allowed ? kAllowed : 0) | + (asleep ? kAsleep : 0) | (uint32_t(rotation) << 3) | (uint32_t(page) << 5); + const uint32_t old = context_.load(); + if ((old & kContextMask) != flags) { + context_ = ((old + kGeneration) & ~kContextMask) | flags; + progress_ = 0; + } +} + +void GestureInput::invalidate() +{ + context_.fetch_add(kGeneration); + progress_ = 0; +} + +bool GestureInput::take(GestureEvent& event, uint64_t now_ms) +{ + if (!queue_ || xQueueReceive(queue_, &event, 0) != pdTRUE) return false; + const uint32_t context = context_.load(); + return event.context == context && (context & (kEnabled | kAllowed)) == (kEnabled | kAllowed) && + now_ms >= event.time_ms && now_ms - event.time_ms <= 300; +} + +void GestureInput::task_entry(void* context) +{ + static_cast(context)->run(); + vTaskDelete(nullptr); +} + +void GestureInput::run() +{ +#if CONFIG_MOSAICO_GESTURE_INPUT + GestureCamera camera; + GestureTracker tracker; + std::unique_ptr detector; + std::unique_ptr recognizer; + uint8_t* rgb = nullptr; + uint32_t previous_context = UINT32_MAX; + bool active = false; + bool fault = false; + bool fatal_cleanup = false; + unsigned capture_failures = 0; + uint64_t next_report = 0; + unsigned measured_frames = 0; + uint64_t measured_us = 0; + unsigned warmup = 0; + + for (;;) { + const uint32_t context = context_.load(); + const bool enabled = context & kEnabled; + const bool allowed = context & kAllowed; + if (context != previous_context) { + tracker.reset(); + progress_ = 0; + previous_context = context; + } + if (!enabled || !allowed || fault) { + if (active || rgb) { + if (!camera.stop()) { + ESP_LOGE(TAG, "camera cleanup failed; recognition disabled until reboot"); + fatal_cleanup = true; + fault = true; + } + recognizer.reset(); + detector.reset(); + heap_caps_free(rgb); + rgb = nullptr; + active = false; + } + progress_ = 0; + if (!enabled && !fatal_cleanup) { fault = false; status_ = GestureStatus::Off; } + else if (fatal_cleanup) status_ = GestureStatus::Fault; + else if (!fault) status_ = GestureStatus::Paused; + vTaskDelay(pdMS_TO_TICKS(100)); + continue; + } + if (!active) { + status_ = GestureStatus::Starting; + // ESP-DL allocates model arenas during construction. Keep a conservative + // margin for both arenas, camera buffers and the concurrently running UI. + if (heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM) < 6 * 1024 * 1024 || + heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT) < 96 * 1024) { + status_ = GestureStatus::MemoryError; + fault = true; + continue; + } + rgb = static_cast(heap_caps_malloc(kGestureImageBytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (!rgb) { status_ = GestureStatus::MemoryError; fault = true; continue; } + if (!camera.start()) { + status_ = GestureStatus::CameraError; + fault = true; + continue; + } + detector.reset(new (std::nothrow) HandDetect(HandDetect::ESPDET_PICO_224_224_HAND, false)); + recognizer.reset(new (std::nothrow) HandGestureRecognizer()); + if (!detector || !recognizer) { status_ = GestureStatus::MemoryError; fault = true; continue; } + detector->set_score_thr(0.5F); + tracker.reset(); + active = true; + capture_failures = 0; + warmup = 3; + status_ = GestureStatus::Ready; + ESP_LOGI(TAG, "local gestures ready; remove hand before starting a new action"); + } + const uint64_t started_us = esp_timer_get_time(); + // Display rotation maps logical pixels to physical coordinates; camera + // observations require the inverse transform after the mounting correction. + const unsigned display_rotation = (context >> 3) & 3; + const unsigned rotation = (CONFIG_MOSAICO_CAMERA_ROTATION / 90 + 4 - display_rotation) % 4; +#ifdef CONFIG_MOSAICO_CAMERA_MIRROR + const bool mirror = true; +#else + const bool mirror = false; +#endif + if (!camera.read_rgb(rgb, rotation, mirror)) { + tracker.reset(); + progress_ = 0; + if (++capture_failures >= 3) { fault = true; status_ = GestureStatus::CameraError; } + vTaskDelay(pdMS_TO_TICKS(20)); + continue; + } + capture_failures = 0; + if (warmup) { --warmup; continue; } + if (context != context_.load()) continue; + dl::image::img_t image{rgb, + static_cast(rotation & 1 ? kGestureImageHeight : kGestureImageWidth), + static_cast(rotation & 1 ? kGestureImageWidth : kGestureImageHeight), + dl::image::DL_IMAGE_PIX_TYPE_RGB888}; + const auto& hands = detector->run(image); + HandObservation observation{}; + observation.count = hands.size(); + if (hands.size() == 1) { + const auto& hand = hands.front(); + if (hand.box.size() == 4) { + observation.x = (hand.box[0] + hand.box[2]) * 0.5F / image.width; + observation.y = (hand.box[1] + hand.box[3]) * 0.5F / image.height; + observation.width = float(hand.box[2] - hand.box[0]) / image.width; + observation.height = float(hand.box[3] - hand.box[1]) / image.height; + observation.score = hand.score; + if (!(context & kAsleep) && hand.score >= 0.65F && + hand.box[0] >= 0 && hand.box[1] >= 0 && hand.box[2] <= image.width && + hand.box[3] <= image.height && hand.box[2] > hand.box[0] && hand.box[3] > hand.box[1]) { + const auto classes = recognizer->recognize(image, hands); + for (const auto& result : classes) { + if (result.cat_name && std::strcmp(result.cat_name, "ok") == 0) + observation.ok_score = result.score; + } + } + } + } + const uint64_t finished_us = esp_timer_get_time(); + if (context != context_.load()) continue; + const GestureAction action = tracker.update(observation, finished_us / 1000); + progress_ = tracker.hold_progress(); + if (action != GestureAction::None) { + const GestureEvent event{action, context, finished_us / 1000}; + xQueueSend(queue_, &event, 0); + } + measured_us += finished_us - started_us; + ++measured_frames; + if (finished_us / 1000 >= next_report) { + ESP_LOGI(TAG, "frames=%u mean_cycle_ms=%llu internal_free=%u psram_free=%u stack_free=%u", + measured_frames, static_cast(measured_us / measured_frames / 1000), + unsigned(heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), + unsigned(heap_caps_get_free_size(MALLOC_CAP_SPIRAM)), unsigned(uxTaskGetStackHighWaterMark(nullptr))); + measured_us = 0; + measured_frames = 0; + next_report = finished_us / 1000 + 10000; + } + vTaskDelay(pdMS_TO_TICKS(20)); + } +#endif +} + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/gesture_input.hpp b/firmware/targets/esp_mosaico/main/gesture_input.hpp new file mode 100644 index 0000000..3a1c7b0 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/gesture_input.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include "freertos/FreeRTOS.h" +#include "freertos/queue.h" +#include "gesture_tracker.hpp" +#include "orientation.hpp" + +namespace usage_panel::mosaico { + +enum class GestureStatus : uint8_t { Off, Starting, Ready, Paused, CameraError, MemoryError, Fault }; + +struct GestureEvent { + GestureAction action = GestureAction::None; + uint32_t context = 0; + uint64_t time_ms = 0; +}; + +/** Worker-owned camera/model pipeline. The application owns context and event consumption. */ +class GestureInput { +public: + bool begin(); + void configure(bool enabled, bool allowed, bool asleep, Page page, ScreenRotation rotation); + /** Cancels pending recognition after another input is accepted. Main task only. */ + void invalidate(); + bool take(GestureEvent& event, uint64_t now_ms); + GestureStatus status() const { return status_.load(); } + uint8_t progress() const { return progress_.load(); } + +private: + static void task_entry(void* context); + void run(); + QueueHandle_t queue_ = nullptr; + std::atomic context_{0}; + std::atomic status_{GestureStatus::Off}; + std::atomic progress_{0}; +}; + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp new file mode 100644 index 0000000..8133072 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp @@ -0,0 +1,99 @@ +#include "gesture_tracker.hpp" + +#include +#include + +namespace usage_panel::mosaico { + +GestureEffect route_gesture(GestureAction action, bool allowed, bool asleep) +{ + if (!allowed || action == GestureAction::None) return GestureEffect::None; + if (action == GestureAction::EnterClock) return asleep ? GestureEffect::None : GestureEffect::Clock; + if (asleep) return GestureEffect::Wake; + return action == GestureAction::SwipeLeft ? GestureEffect::Next : GestureEffect::Previous; +} + +void GestureTracker::reset() +{ + const uint64_t last_action = last_action_; + *this = GestureTracker{}; + last_action_ = last_action; +} + +GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) +{ + progress_ = 0; + if (h.count == 0) { + if (!absent_) { absent_ = true; absent_since_ = now; } + if (now - absent_since_ >= 300 && now - last_action_ >= 500) + waiting_release_ = false; + tracking_ = false; + holding_ok_ = false; + return GestureAction::None; + } + absent_ = false; + const bool valid = h.count == 1 && std::isfinite(h.x) && std::isfinite(h.y) && + std::isfinite(h.width) && std::isfinite(h.height) && std::isfinite(h.score) && + h.score >= 0.65F && h.x >= 0 && h.x <= 1 && h.y >= 0 && h.y <= 1 && + h.width >= 0.06F && h.height >= 0.06F && h.width <= 0.9F && h.height <= 0.95F; + if (!valid) { + tracking_ = false; + holding_ok_ = false; + waiting_release_ = true; + } + if (waiting_release_) return GestureAction::None; + + if (tracking_ && (now < last_sample_ || now - last_sample_ > 250 || + std::fabs(h.x - previous_.x) > std::max(0.18F, (h.width + previous_.width) * 0.75F) || + std::fabs(h.y - previous_.y) > std::max(0.18F, (h.height + previous_.height) * 0.75F))) { + reset(); + return GestureAction::None; + } + if (!tracking_) { + tracking_ = true; + started_ = now; + samples_ = 0; + start_x_ = h.x; + start_y_ = h.y; + path_x_ = max_dy_ = 0; + previous_ = h; + } + ++samples_; + path_x_ += std::fabs(h.x - previous_.x); + max_dy_ = std::max(max_dy_, std::fabs(h.y - start_y_)); + const float dx = h.x - start_x_; + previous_ = h; + last_sample_ = now; + + GestureAction action = GestureAction::None; + if (samples_ >= 3 && now - started_ >= 150 && now - started_ <= 900 && + std::fabs(dx) >= 0.25F && std::fabs(dx) >= 2 * max_dy_ && + max_dy_ <= 0.15F && std::fabs(dx) >= path_x_ * 0.75F) { + action = dx < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight; + } + + if (std::isfinite(h.ok_score) && h.ok_score >= 0.85F) { + if (!holding_ok_ || std::fabs(h.x - ok_x_) > 0.08F || std::fabs(h.y - ok_y_) > 0.08F) { + holding_ok_ = true; + ok_since_ = now; + ok_x_ = h.x; + ok_y_ = h.y; + } + progress_ = static_cast(std::min(100, (now - ok_since_) / 5)); + if (now - ok_since_ >= 500 && samples_ >= 3) action = GestureAction::EnterClock; + } else { + holding_ok_ = false; + } + if (action != GestureAction::None) { + waiting_release_ = true; + last_action_ = now; + tracking_ = holding_ok_ = false; + progress_ = 0; + } else if (now - started_ > 900 && !holding_ok_) { + // A stationary hand may begin a later deliberate swipe without disappearing. + tracking_ = false; + } + return action; +} + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp new file mode 100644 index 0000000..55de43a --- /dev/null +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include + +namespace usage_panel::mosaico { + +enum class GestureAction : uint8_t { None, SwipeLeft, SwipeRight, EnterClock }; +enum class GestureEffect : uint8_t { None, Wake, Previous, Next, Clock }; + +/** Applies application input policy before any page or screensaver mutation. */ +GestureEffect route_gesture(GestureAction action, bool allowed, bool asleep); + +/** A detection in normalized, upright screen coordinates; count includes all hands. */ +struct HandObservation { + unsigned count = 0; + float x = 0; + float y = 0; + float width = 0; + float height = 0; + float score = 0; + float ok_score = 0; +}; + +/** Time-based single-hand recognition. A completed action requires release before reuse. */ +class GestureTracker { +public: + GestureAction update(const HandObservation& hand, uint64_t now_ms); + /** Invalidates prior observations and requires fresh, sustained absence of hands. */ + void reset(); + uint8_t hold_progress() const { return progress_; } + +private: + bool waiting_release_ = true; + bool absent_ = false; + bool tracking_ = false; + bool holding_ok_ = false; + uint64_t absent_since_ = 0; + uint64_t last_action_ = 0; + uint64_t started_ = 0; + uint64_t last_sample_ = 0; + uint64_t ok_since_ = 0; + unsigned samples_ = 0; + uint8_t progress_ = 0; + HandObservation previous_{}; + float start_x_ = 0; + float start_y_ = 0; + float path_x_ = 0; + float max_dy_ = 0; + float ok_x_ = 0; + float ok_y_ = 0; +}; + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/idf_component.yml b/firmware/targets/esp_mosaico/main/idf_component.yml index de5196b..f675b62 100644 --- a/firmware/targets/esp_mosaico/main/idf_component.yml +++ b/firmware/targets/esp_mosaico/main/idf_component.yml @@ -4,3 +4,6 @@ dependencies: # Keeping both dependencies in the first pass preserves the locked graph. espressif/esp_lvgl_adapter: "0.6.2" lvgl/lvgl: "^9.5.0,<9.6.0" + espressif/esp_video: "2.5.0" + espressif/hand_detect: "0.2.0" + espressif/hand_gesture_recognition: "0.2.0" diff --git a/firmware/targets/esp_mosaico/main/panel_settings.cpp b/firmware/targets/esp_mosaico/main/panel_settings.cpp new file mode 100644 index 0000000..5c310ec --- /dev/null +++ b/firmware/targets/esp_mosaico/main/panel_settings.cpp @@ -0,0 +1,40 @@ +#include "panel_settings.hpp" +#include "nvs.h" + +namespace usage_panel::mosaico { + +PanelSettings load_panel_settings(DisplaySettings defaults) +{ + PanelSettings result{load_display_settings(defaults), false}; + nvs_handle_t handle; + if (nvs_open("display", NVS_READONLY, &handle) == ESP_OK) { + uint8_t enabled = 0; + if (nvs_get_u8(handle, "gestures", &enabled) == ESP_OK && enabled <= 1) + result.gestures = enabled != 0; + nvs_close(handle); + } + return result; +} + +esp_err_t save_panel_settings(const PanelSettings& settings) +{ + if (settings.brightness < 1 || settings.brightness > 100 || settings.clock_timeout_seconds > 86400) + return ESP_ERR_INVALID_ARG; + nvs_handle_t handle; + esp_err_t error = nvs_open("display", NVS_READWRITE, &handle); + if (error != ESP_OK) return error; + error = nvs_set_u8(handle, "brightness", settings.brightness); + if (error == ESP_OK) error = nvs_set_u32(handle, "clock_timeout", settings.clock_timeout_seconds); + if (error == ESP_OK) error = nvs_set_u8(handle, "gestures", settings.gestures); + if (error == ESP_OK) error = nvs_commit(handle); + nvs_close(handle); + return error; +} + +bool settings_differ(const PanelSettings& a, const PanelSettings& b) +{ + return a.brightness != b.brightness || a.clock_timeout_seconds != b.clock_timeout_seconds || + a.gestures != b.gestures; +} + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/panel_settings.hpp b/firmware/targets/esp_mosaico/main/panel_settings.hpp new file mode 100644 index 0000000..8fb91da --- /dev/null +++ b/firmware/targets/esp_mosaico/main/panel_settings.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include "usage_panel_state/display_settings.hpp" + +namespace usage_panel::mosaico { + +struct PanelSettings : DisplaySettings { + bool gestures = false; +}; + +PanelSettings load_panel_settings(DisplaySettings defaults); +/** Stores display and local input preferences through a single NVS handle. */ +esp_err_t save_panel_settings(const PanelSettings& settings); +bool settings_differ(const PanelSettings& a, const PanelSettings& b); + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/partitions.csv b/firmware/targets/esp_mosaico/partitions.csv index a3eb613..2e32cad 100644 --- a/firmware/targets/esp_mosaico/partitions.csv +++ b/firmware/targets/esp_mosaico/partitions.csv @@ -1,6 +1,6 @@ # Name, Type, SubType, Offset, Size nvs, data, nvs, 0x9000, 0x6000 phy_init, data, phy, 0xf000, 0x1000 -ota_0, app, ota_0, 0x10000, 0x400000 -ota_1, app, ota_1, 0x410000, 0x400000 +ota_0, app, ota_0, 0x10000, 0x700000 otadata, data, ota, 0x810000, 0x2000 +ota_1, app, ota_1, 0x820000, 0x700000 diff --git a/firmware/targets/esp_mosaico/sdkconfig.defaults b/firmware/targets/esp_mosaico/sdkconfig.defaults index b380051..59a06bd 100644 --- a/firmware/targets/esp_mosaico/sdkconfig.defaults +++ b/firmware/targets/esp_mosaico/sdkconfig.defaults @@ -63,5 +63,31 @@ CONFIG_ESP_DESKTOP_BUDDY_RX_INBOX_DEPTH=32 CONFIG_ESP_DESKTOP_BUDDY_TX_QUEUE_DEPTH=4 CONFIG_ESP_DESKTOP_BUDDY_FOLDER_PUSH_MAX_RAW_CHUNK=4096 CONFIG_ESP_DESKTOP_BUDDY_FOLDER_PUSH_MAX_DECODED_CHUNK=2880 -CONFIG_ESP_DESKTOP_BUDDY_FOLDER_PUSH_MAX_TRANSFER_BYTES=4194816 +CONFIG_ESP_DESKTOP_BUDDY_FOLDER_PUSH_MAX_TRANSFER_BYTES=7340544 CONFIG_ESP_DESKTOP_BUDDY_FOLDER_PUSH_FLAT_PATHS_ONLY=y + +# CameraBoard D2 occupies the USB Serial/JTAG pad; retain UART diagnostics. +CONFIG_ESP_CONSOLE_SECONDARY_NONE=y +CONFIG_ESP_VIDEO_ENABLE_DVP_VIDEO_DEVICE=y +CONFIG_CAMERA_OV3640=y +CONFIG_CAMERA_OV3640_DVP_YUV422_UYVY_640X480_30FPS=y +CONFIG_CAMERA_OV3640_DVP_DEFAULT_FMT_YUV422_UYVY_640X480_30FPS=y +CONFIG_MOSAICO_GESTURE_INPUT=y +CONFIG_MOSAICO_CAMERA_ROTATION=180 +CONFIG_MOSAICO_CAMERA_MIRROR=y + +# The vision pipeline supplies RGB888; other ESP-DL converters are unused. +# CONFIG_PIX_CVT_RGB565_TO_RGB565_SUPPORT is not set +# CONFIG_PIX_CVT_RGB565_TO_RGB888_SUPPORT is not set +# CONFIG_PIX_CVT_RGB565_TO_GRAY_SUPPORT is not set +# CONFIG_PIX_CVT_RGB565_TO_HSV_SUPPORT is not set +# CONFIG_PIX_CVT_RGB888_TO_RGB565_SUPPORT is not set +# CONFIG_PIX_CVT_RGB888_TO_GRAY_SUPPORT is not set +# CONFIG_PIX_CVT_RGB888_TO_HSV_SUPPORT is not set +# CONFIG_PIX_CVT_GRAY_TO_GRAY_SUPPORT is not set +# CONFIG_PIX_CVT_HSV_TO_HSV_MASK_SUPPORT is not set +# CONFIG_PIX_CVT_YUV_TO_RGB888_SUPPORT is not set +# CONFIG_PIX_CVT_YUV_TO_RGB565_SUPPORT is not set +# CONFIG_PIX_CVT_YUV_TO_GRAY_SUPPORT is not set +# CONFIG_PIX_CVT_YUV_TO_HSV_SUPPORT is not set +# CONFIG_PIX_CVT_YUV_TO_YUV_SUPPORT is not set diff --git a/firmware/targets/esp_mosaico/test_apps/logic/main/CMakeLists.txt b/firmware/targets/esp_mosaico/test_apps/logic/main/CMakeLists.txt index 0a70c1a..e658d57 100644 --- a/firmware/targets/esp_mosaico/test_apps/logic/main/CMakeLists.txt +++ b/firmware/targets/esp_mosaico/test_apps/logic/main/CMakeLists.txt @@ -3,8 +3,11 @@ idf_component_register( "test_main.cpp" "test_local_clock.cpp" "test_orientation.cpp" + "test_gestures.cpp" "../../../main/local_clock.cpp" "../../../main/orientation.cpp" + "../../../main/gesture_tracker.cpp" + "../../../main/gesture_image.cpp" INCLUDE_DIRS "../../../main" REQUIRES unity usage_protocol usage_panel_state WHOLE_ARCHIVE diff --git a/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp b/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp new file mode 100644 index 0000000..8d9c54a --- /dev/null +++ b/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp @@ -0,0 +1,106 @@ +#include "gesture_tracker.hpp" +#include "gesture_image.hpp" +#include "unity.h" + +#include +#include + +using namespace usage_panel::mosaico; + +namespace { +HandObservation hand(float x, float y = 0.5F, float ok = 0) +{ + return {1, x, y, 0.25F, 0.35F, 0.9F, ok}; +} +void arm(GestureTracker& tracker) +{ + tracker.update({}, 1000); + tracker.update({}, 1400); +} +} + +TEST_CASE("swipes require release and issue exactly one directional command", "[gesture]") +{ + for (int direction : {-1, 1}) { + GestureTracker tracker; + arm(tracker); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F), 1500))); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F + direction * 0.15F), 1600))); + const auto expected = direction < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight; + TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(hand(0.5F + direction * 0.3F), 1700))); + for (unsigned t = 1800; t < 3000; t += 100) + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F), t))); + tracker.update({}, 3000); + tracker.update(hand(0.5F), 3100); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.8F), 3300))); + tracker.update({}, 3400); + tracker.update({}, 3800); + tracker.update(hand(0.3F), 3900); + tracker.update(hand(0.45F), 4000); + TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.6F), 4100))); + } +} + +TEST_CASE("OK requires a stable hold and release after context invalidation", "[gesture]") +{ + GestureTracker tracker; + arm(tracker); + for (unsigned t = 1500; t < 2000; t += 100) + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F, 0.5F, 0.95F), t))); + TEST_ASSERT_EQUAL_INT(80, tracker.hold_progress()); + TEST_ASSERT_EQUAL_INT(int(GestureAction::EnterClock), int(tracker.update(hand(0.5F, 0.5F, 0.95F), 2000))); + tracker.reset(); + for (unsigned t = 2100; t <= 3000; t += 100) + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F, 0.5F, 0.95F), t))); +} + +TEST_CASE("vertical motion jitter ambiguous hands and stale samples do not trigger", "[gesture]") +{ + for (unsigned scenario = 0; scenario < 5; ++scenario) { + GestureTracker tracker; + arm(tracker); + for (unsigned i = 0; i < 7; ++i) { + HandObservation observation = hand(0.2F + i * 0.08F); + unsigned time = 1500 + i * 100; + if (scenario == 0) { observation.x = 0.5F; observation.y = 0.2F + i * 0.1F; } + if (scenario == 1) observation.x = i % 2 ? 0.55F : 0.45F; + if (scenario == 2) observation.count = 2; + if (scenario == 3) time = 1500 + i * 400; + if (scenario == 4) observation.x = std::numeric_limits::quiet_NaN(); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(observation, time))); + } + } +} + +TEST_CASE("wake consumes the first swipe and protected states reject all commands", "[gesture]") +{ + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeLeft, true, true))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeRight, true, true))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::SwipeLeft, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Previous), int(route_gesture(GestureAction::SwipeRight, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Clock), int(route_gesture(GestureAction::EnterClock, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::None), int(route_gesture(GestureAction::EnterClock, true, true))); + for (auto action : {GestureAction::SwipeLeft, GestureAction::SwipeRight, GestureAction::EnterClock}) + TEST_ASSERT_EQUAL_INT(int(GestureEffect::None), int(route_gesture(action, false, false))); +} + +TEST_CASE("UYVY conversion respects padded stride mirror and four rotations", "[gesture]") +{ + // Four grayscale quadrants plus row padding make the geometric mapping observable. + const uint8_t source[] = {128, 16, 128, 80, 255, 255, 128, 160, 128, 235, 255, 255}; + static std::array output; + const unsigned expected[4][4] = {{0, 74, 167, 255}, {167, 0, 255, 74}, + {255, 167, 74, 0}, {74, 255, 0, 167}}; + for (unsigned rotation = 0; rotation < 4; ++rotation) { + const unsigned w = rotation & 1 ? kGestureImageHeight : kGestureImageWidth; + const unsigned h = rotation & 1 ? kGestureImageWidth : kGestureImageHeight; + TEST_ASSERT_TRUE(prepare_gesture_image(source, sizeof(source), 2, 2, 6, output.data(), rotation, false)); + const unsigned pixels[] = {0, w - 1, (h - 1) * w, h * w - 1}; + for (unsigned i = 0; i < 4; ++i) + TEST_ASSERT_INT_WITHIN(1, expected[rotation][i], output[pixels[i] * 3]); + } + TEST_ASSERT_TRUE(prepare_gesture_image(source, sizeof(source), 2, 2, 6, output.data(), 0, true)); + TEST_ASSERT_INT_WITHIN(1, 74, output[0]); + TEST_ASSERT_FALSE(prepare_gesture_image(source, 4, 2, 2, 6, output.data(), 0, false)); + TEST_ASSERT_FALSE(prepare_gesture_image(source, sizeof(source), 3, 2, 6, output.data(), 0, false)); +} diff --git a/firmware/test_apps/panel_state/main/test_screensaver_state.cpp b/firmware/test_apps/panel_state/main/test_screensaver_state.cpp index 55c19bb..29a2673 100644 --- a/firmware/test_apps/panel_state/main/test_screensaver_state.cpp +++ b/firmware/test_apps/panel_state/main/test_screensaver_state.cpp @@ -4,6 +4,20 @@ using namespace usage_panel; +TEST_CASE("explicit clock entry is idempotent and wakes to the original page", "[screensaver]") +{ + ScreensaverController controller(0); + controller.reset(0, Page::Codex); + controller.enter_clock(1000, Page::Claude); + controller.enter_clock(2000, Page::Overview); + TEST_ASSERT_TRUE(controller.view(2000).active); + TEST_ASSERT_EQUAL_INT(int(Page::Claude), int(controller.restore_page())); + controller.note_touch_down(3000); + TEST_ASSERT_FALSE(controller.view(3000).active); + TEST_ASSERT_TRUE(controller.view(3000).consume_touch); + TEST_ASSERT_EQUAL_INT(int(Page::Claude), int(controller.restore_page())); +} + TEST_CASE("screensaver enters after five minutes and preserves page", "[screensaver]") { ScreensaverController controller; From c227abbc01c6fe1358d2f3d6e45eb8735a9015f0 Mon Sep 17 00:00:00 2001 From: wulu Date: Wed, 23 Sep 2026 11:45:49 +0800 Subject: [PATCH 02/15] docs(mosaico): index gesture reference repos --- docs/design/mosaico-gesture-input.md | 2 ++ .../targets/esp_mosaico/CAMERA_UPSTREAM.md | 24 +++++++++++++++++++ firmware/targets/esp_mosaico/README.md | 2 ++ 3 files changed, 28 insertions(+) diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md index a93edaa..273e86a 100644 --- a/docs/design/mosaico-gesture-input.md +++ b/docs/design/mosaico-gesture-input.md @@ -166,6 +166,8 @@ eim run "idf.py -C firmware/test_apps/panel_state build" v6.1 ## 参考来源 +后续开发优先阅读[摄像头与手势仓库参考索引](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)：集中记录官方仓库、项目 fork、固定提交/组件版本、各仓库用途及本地代码入口。下列默认分支链接用于查阅，不代表本实现的锁定版本。 + - [分享报告及交互修订](https://chatgpt.com/share/6ab29265-faa8-83e8-8e32-fa2bb8b8989b)；另一份为本次任务内粘贴的补充报告。 - [ESP-DL Model Zoo](https://github.com/espressif/esp-dl/blob/master/models/README.md) - [HandDetect CMake](https://github.com/espressif/esp-dl/blob/master/models/hand_detect/CMakeLists.txt) 与 [模型说明](https://github.com/espressif/esp-dl/blob/master/models/hand_detect/README.md) diff --git a/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md b/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md index c161103..5026daf 100644 --- a/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md +++ b/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md @@ -1,5 +1,29 @@ # Camera and gesture dependencies +## Repository reference map + +Start here when continuing camera or gesture development. Repository default +branches are browsing references; reproduce this integration using the pinned +commits below and [dependencies.lock](dependencies.lock), not the latest branch. + +| Repository | What to reference | Integration baseline | +| --- | --- | --- | +| [esp-mosaico-bsp](https://github.com/esp-mosaico/esp-mosaico-bsp) | Board power, shared I2C, expansion slots and display ownership | Commit `392860b1d1a123c3377947074b2af1f600e86c5d`; preserve the local changes in [BSP UPSTREAM.md](../../components/esp-mosaico-bsp/UPSTREAM.md) | +| [esp-mosaico-claw upstream](https://github.com/esp-mosaico/esp-mosaico-claw) / [project fork](https://github.com/eMUQI/esp-mosaico-claw) | Left-slot OV3640 wiring and USB Serial/JTAG pad arbitration | [Committed subboard source](https://github.com/eMUQI/esp-mosaico-claw/blob/903fe11ec3ca4a8881afef104e53a8da040896ac/components/subboard_support/subboard.c); local uncommitted camera changes were not imported | +| [esp-video-components](https://github.com/espressif/esp-video-components) | `esp_video` DVP/V4L2 capture, buffer lifetime and `esp_cam_sensor` OV3640 modes | Registry packages `esp_video` 2.5.0 and `esp_cam_sensor` 2.6.0 | +| [ESP-DL](https://github.com/espressif/esp-dl) | [HandDetect](https://github.com/espressif/esp-dl/tree/master/models/hand_detect), [HandGestureRecognizer](https://github.com/espressif/esp-dl/tree/master/models/hand_gesture_recognition), preprocessing and model selection | Registry packages `esp-dl` 3.3.11, `hand_detect` 0.2.0 and `hand_gesture_recognition` 0.2.0; model digests below | +| [ESP-VISION](https://github.com/espressif/esp-vision) | Mosaico board port and Camera → Image → AI → Display pipeline patterns | Architecture reference only; not a dependency and no source snapshot imported | +| [ESP-WHO](https://github.com/espressif/esp-who) | Capture/inference task separation and visual application lifecycle | Architecture reference only; not the starting point for this Mosaico target | + +## Continuing this implementation + +- Read the [interaction design and implementation record](../../../docs/design/mosaico-gesture-input.md) for swipe direction, wake-only behavior, OK hold and protected states. +- Start in [gesture_camera.cpp](main/gesture_camera.cpp) for capture/pin ownership, [gesture_image.cpp](main/gesture_image.cpp) for image orientation, [gesture_tracker.cpp](main/gesture_tracker.cpp) for temporal recognition, and [gesture_input.cpp](main/gesture_input.cpp) for the worker lifecycle. [app_main.cpp](main/app_main.cpp) owns action routing and UI state. +- Follow the [installation and hardware acceptance guide](../../../docs/validation/mosaico-gestures.md). Software builds/tests passed at the initial implementation; camera orientation, actual recognition, UI layout, resource use and power remain hardware acceptance items. +- Preserve the 7 MiB dual-slot migration contract, default-off setting, local-only image processing and existing BSP display fixes. Recheck the current tree and lockfile before changing dependencies; do not edit generated `managed_components` as the source of truth. + +## Locked component versions + The target uses registry components pinned by `dependencies.lock`: | Component | Version | Role | diff --git a/firmware/targets/esp_mosaico/README.md b/firmware/targets/esp_mosaico/README.md index d80c543..9dec2df 100644 --- a/firmware/targets/esp_mosaico/README.md +++ b/firmware/targets/esp_mosaico/README.md @@ -38,6 +38,8 @@ eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 ### 本地手势输入 +后续开发入口：[参考仓库、固定版本与代码导航](CAMERA_UPSTREAM.md#repository-reference-map)；[交互设计与实施记录](../../../docs/design/mosaico-gesture-input.md)。 + 左槽 OV3640 CameraBoard 可选开启本地手势：左挥下一页、右挥上一页，时钟中首挥只恢复原页；👌 OK 稳定保持约 500 ms 进入时钟。设置中的 `HAND GESTURES` 默认关闭，需保存后启用。图像仅在本机处理。 模型与代码内嵌同一应用，首次从旧 4 MiB 分区升级必须有线迁移。保留设置时使用分段烧录，不能用填充后的完整镜像替代。依赖来源见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)，安装、恢复与实机验证见[手势验收指南](../../../docs/validation/mosaico-gestures.md)。当前仅通过软件测试和构建；相机方向、识别率、性能、功耗和提示布局仍待设备验收。 From 972bec7b54c6e8770a64ece31771d920afbfc3ed Mon Sep 17 00:00:00 2001 From: Wulu Date: Wed, 23 Sep 2026 16:01:47 +0800 Subject: [PATCH 03/15] feat(mosaico): refine camera gestures and toggle preview --- docs/design/amoled-ui.md | 2 + docs/design/mosaico-gesture-input.md | 28 +-- docs/validation/mosaico-gestures.md | 111 ++++++---- .../targets/esp_mosaico/CAMERA_UPSTREAM.md | 41 +++- firmware/targets/esp_mosaico/README.md | 4 +- .../targets/esp_mosaico/main/CMakeLists.txt | 2 +- .../esp_mosaico/main/Kconfig.projbuild | 20 +- .../targets/esp_mosaico/main/app_main.cpp | 70 ++++-- .../esp_mosaico/main/camera_setting.cpp | 31 +++ .../esp_mosaico/main/camera_setting.hpp | 12 + .../targets/esp_mosaico/main/display_ui.cpp | 208 +++++++++++++++--- .../targets/esp_mosaico/main/display_ui.hpp | 41 +++- .../esp_mosaico/main/gesture_camera.cpp | 118 +++++++--- .../esp_mosaico/main/gesture_camera.hpp | 13 +- .../esp_mosaico/main/gesture_image.cpp | 34 ++- .../esp_mosaico/main/gesture_image.hpp | 13 +- .../esp_mosaico/main/gesture_input.cpp | 97 +++++++- .../esp_mosaico/main/gesture_input.hpp | 12 + .../esp_mosaico/main/gesture_tracker.cpp | 65 ++++-- .../esp_mosaico/main/gesture_tracker.hpp | 4 +- .../esp_mosaico/main/panel_settings.cpp | 40 ---- .../esp_mosaico/main/panel_settings.hpp | 16 -- .../targets/esp_mosaico/sdkconfig.defaults | 13 +- .../test_apps/logic/main/test_gestures.cpp | 154 ++++++++++--- 24 files changed, 864 insertions(+), 285 deletions(-) create mode 100644 firmware/targets/esp_mosaico/main/camera_setting.cpp create mode 100644 firmware/targets/esp_mosaico/main/camera_setting.hpp delete mode 100644 firmware/targets/esp_mosaico/main/panel_settings.cpp delete mode 100644 firmware/targets/esp_mosaico/main/panel_settings.hpp diff --git a/docs/design/amoled-ui.md b/docs/design/amoled-ui.md index 6356396..ce08be2 100644 --- a/docs/design/amoled-ui.md +++ b/docs/design/amoled-ui.md @@ -72,6 +72,7 @@ Waveshare：屏幕边距 24px；卡片宽 432px；卡片内边距 20–22px； - **底部导航**：方屏为**纯文字标签 + 下划线指示**，不用色块、不用卡片底。34px 高的三等分标签条，标签 13px 600 大写、字距 0.06em；未选中 `#565D68`，选中 `#C7CCD4` 并在标签下方 7px 处加一条 26×2px（圆角 1px）`#C7CCD4` 短横。导航是次要信息，任何情况下都不能成为屏幕上最亮的元素。**下划线是三个标签共用的同一条**，翻页时以 180ms 缓出横向滑到新标签下方，而不是在旧位置熄灭、新位置点亮；页面本身仍是硬切换。 - **连接状态**：右上角一个 8px 状态色圆点 + 14px `#C7CCD4` 文字。不显示 `SECURE` 之类无操作价值的标签。 - **电池**：连接状态右侧，30×14 描边（1px `#565D68`）+ 2px 正极点，内含 4 段。**分级显示，不显示百分比**：FULL=4 段、HIGH=3 段、MID=2 段、LOW=1 段、无数据=0 段；点亮色 `#C7CCD4`，熄灭段 `#2C313B`；仅 LOW 时点亮段转 `#F5B84C`。**充电**：外框左侧 gap 5px 有 7×12 闪电槽，**始终占位**（不充电时留空，绝不收起，否则插拔 USB 会让 `LINKED` 平移）；充电时填 `#C7CCD4`，段数照常反映电量，不做动画；充电时 LOW 不再变黄（外部供电已接入，此时低电量告警不再提供操作价值），屏保 `LOW` 角标同时撤掉。USB 只是供电，不改变 `LINKED`，界面不出现 `USB` 标签。 +- **摄像头标识**（仅 Mosaico）：`CAMERA` 开关已保存为打开时，连接状态点左侧 10px 显示 26×16 实心相机剪影（顶部 8×4 取景凸起，bg 色镜头环内含 4px 镜头点），与状态点垂直居中。采集中（启动或识别）为 `text` 色；开关打开但采集暂停（设置、配对、OTA）为 `dim`；采集失败时为 `dim` 并叠加 `#C7CCD4` 斜杠（外圈 bg 色描边）。开关关闭时不占位。顶栏不显示手势文字提示，仅保留 OK 停留进度条。保持灰阶，不使用 good/warn/bad 色。屏保不显示。 - **时间**：顶栏左侧，22px 700，`text` 色，24 小时制不显示秒。页面归属由底部导航指示，无需重复标题。硬件无温湿度传感器，界面不出现环境信息。 - **屏保态（SCREENSAVER）**：空闲 5 分钟后进入，触摸屏幕或晃动（IMU）唤醒，回到进入前的页面。规则是**整体降一级**——时钟 `muted #8B93A1`（120px，居中 top 158）、日期一行 20px `dim`、顶栏与底部导航全部隐藏、卡片去掉 panel 底与描边（纯 `bg`，AMOLED 省电）。屏幕上不得出现 `text` 级亮度。底部 13px `#2C313B` 提示 `TAP OR SHAKE TO WAKE`，进入 3 秒后淡出。仅电池 LOW 时右上角显示降亮的 `LOW` + 电池图标（`#8A6A2E`），其余状态一概不显示。防烧屏：整块内容每 60 秒随机位移 ±10px。 @@ -84,6 +85,7 @@ Waveshare：屏幕边距 24px；卡片宽 432px；卡片内边距 20–22px； - **亮度设置行**：复用现有 48px 数值字体；右侧加减按钮为 80×80px，间隔 8px。仅通过按钮调整，不显示不可操作的滑条。到达 1% 或 100% 时禁用对应按钮，边界不循环。 - **有限选项选择**：AUTO CLOCK 下方标明 AFTER IDLE / MIN；OFF、1、5、10、30 五项为 80×64px，间隔 8px，一次点击选定。只有当前选项反色，不累计填充；OFF 不拼接 MIN。保留非标准 menuconfig 默认时长，未匹配预设时显示当前秒数，直到用户主动选择预设。 - **操作状态**：16px 圆角，可见边界与点击区域一致。选中项及可用 SAVE 使用 text 底、bg 字；未选项和 CANCEL 使用 panel 底、muted 字。按下时背景和边框产生灰阶反馈；禁用控件使用 panel 底、dim 字，不响应点击。不使用主题阴影或过渡。 +- **开关**（仅 Mosaico `CAMERA`）：独立一行，与上方以分隔线隔开；右侧 80×44px 胶囊开关，与加减按钮右对齐。关闭为 panel 底、muted 圆钮居左；打开为反色 text 底、bg 圆钮居右。与亮度、自动时钟一样只是预览：切换后启用 SAVE，SAVE 一并写入 NVS，CANCEL 恢复已保存值。 - **保存与取消**：底部按钮均为 206×64px，间隔 20px。未修改时禁用 SAVE，修改后启用；CANCEL 始终可用。保存成功后退出，失败时保留页面和预览，在按钮上方显示 SAVE FAILED - TRY AGAIN 并允许重试。不常驻显示持久化实现说明。 实机验证需覆盖最低亮度恢复操作、桌面支架姿态下的按钮命中、OFF、未修改／已修改以及保存失败状态；像素尺寸与桌面预览不能替代触控和 AMOLED 可读性验收。 diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md index 273e86a..12a635b 100644 --- a/docs/design/mosaico-gesture-input.md +++ b/docs/design/mosaico-gesture-input.md @@ -4,7 +4,7 @@ ## 目标与范围 -在 ESP-Mosaico 上增加可选的本地摄像头输入，保留触摸、AI 键、BLE 和自动时钟屏保。先验证 CameraBoard → HandDetect → 横向轨迹 → 翻页/唤醒，再增加现成分类器的 `ok` 手势进入时钟。 +在 ESP-Mosaico 上增加可选的本地摄像头输入，保留触摸、AI 键、BLE 和自动时钟屏保。先验证 CameraBoard → HandDetect → 四向轨迹 → 翻页/唤醒，再增加现成分类器的 `ok` 手势进入时钟。 摄像头图像仅在设备内处理，不上传、不默认保存、不显示常驻预览。第一阶段不训练模型、不引入 ESP-VISION/MicroPython 或 ESP-WHO 整套应用、不扩展 Bridge 协议、不把视觉依赖加入其他板卡。识别区域内出现手不等于用户发出命令。 @@ -12,18 +12,20 @@ ## 两份报告的取舍 -已阅读分享对话和补充报告。采用分享对话最后确认的“挥手翻页/唤醒”语义；👌 OK 进入时钟作为第二阶段能力。两份报告的方向描述相反，以当前触摸行为为准。 +已阅读分享对话和补充报告。采用分享对话最后确认的“挥手翻页/唤醒”语义；👌 OK 进入时钟作为第二阶段能力。空中挥手按导航项方向切页，触摸滑动保持原有语义。 | 当前状态 | 有效动作 | 行为 | | --- | --- | --- | -| 正常用量页 | 向左挥手 | 下一页 | -| 正常用量页 | 向右挥手 | 上一页 | -| 时钟屏保 | 任一有效横向挥手 | 只恢复原页面；本轮不再翻页 | +| 正常用量页 | 向左挥手 | 上一页 | +| 正常用量页 | 向右挥手 | 下一页 | +| 正常用量页 | 向上挥手 | 上一页 | +| 正常用量页 | 向下挥手 | 下一页 | +| 时钟屏保 | 任一有效四向挥手 | 只恢复原页面；本轮不再翻页 | | 正常用量页 | 稳定 `ok` | 进入时钟并保存原页面 | | 时钟屏保 | `ok` | 无操作，不执行 Toggle | | 设置、配对确认、OTA | 任意手势 | 丢弃，不延后执行 | -左右方向均为用户面对当前屏幕所见的方向。挥动判定只用 HandDetect，因此不能宣称识别了“张掌”；若日常活动误触发无法达标，再评估以 `five` 分类作为挥动的手形门控。 +上下左右方向均为用户面对当前屏幕所见的方向。挥动判定只用 HandDetect，因此不能宣称识别了“张掌”；若日常活动误触发无法达标，再评估以 `five` 分类作为挥动的手形门控。 补充报告中“看到手/普通运动即可唤醒”暂不采纳，避免桌前活动持续打断屏保。ROI motion 仅是后续功耗优化候选，不能代替有效挥手确认。 @@ -66,7 +68,7 @@ CameraBoard / esp_video → 获取最新帧、格式转换 → HandDetect → 屏幕坐标中的单手轨迹 - → SwipeLeft / SwipeRight / EnterClock + → SwipeLeft / SwipeRight / SwipeUp / SwipeDown / EnterClock → Mosaico 主循环检查状态 → ScreensaverController / page_after_swipe / ui.request_page ``` @@ -86,11 +88,11 @@ CameraBoard / esp_video 状态从 `Idle → Tracking → WaitRelease → Idle`： 1. 单个可信目标稳定出现才开始跟踪；以位置连续性和 bbox 尺寸约束保持目标关联。 -2. 在限定时间内检查净横向位移、累计路径方向一致性、纵向漂移、置信度与合理 bbox 大小。突然跳变、长时间漏检、模糊的多手关联取消候选。 +2. 在限定时间内检查主轴净位移、累计路径方向一致性、垂直于主轴的漂移、置信度与合理 bbox 大小。突然跳变、长时间漏检、模糊的多手关联取消候选。 3. 完成一次动作只发一个事件，随后等待持续无手且最短冷却已结束再重新接受输入。 4. 单帧漏检、分类器 `no_hand` 或 worker 暂停不能单独作为“手已离开”的证据。恢复采集后要观察新鲜帧，禁止旧事件回放。 -以下仅为试验起点，不是已验证的产品参数：横向净位移约画面宽度 25%，动作窗口 150–900 ms，横向位移至少为纵向的 2 倍；连续无手约 300 ms，最短冷却约 500 ms；事件有效期约 300 ms。阈值结合有效帧数、视场、距离和漏检率一起校准。持续可见的手不因冷却超时而重复翻页。 +以下仅为试验起点，不是已验证的产品参数：主轴净位移约对应画面宽度或高度的 25%，动作窗口 150–900 ms，主轴净位移至少为另一轴最大偏移的 2 倍；连续无手约 300 ms，最短冷却约 500 ms；事件有效期约 300 ms。阈值结合有效帧数、视场、距离和漏检率一起校准。持续可见的手不因冷却超时而重复翻页。 ## 分阶段工作与完成条件 @@ -107,11 +109,11 @@ CameraBoard / esp_video **完成条件：**可复现的依赖与构建、可用的实时 bbox、资源与延迟记录、正常停止/缺摄像头降级。只构建成功不算实时识别通过。当前已完成软件接入及 UI，以便一次烧录完成端到端验收；实时识别和资源指标未通过实机验证前，不视为产品验收完成。 -### P1：左右挥手翻页与屏保唤醒 +### P1：四向挥手翻页与屏保唤醒 - [ ] 完成纯轨迹判定和坐标映射；原型先验证固定朝向，再验证四个朝向。 - [ ] 接入主循环的 Swipe/Wake 策略、一次性消费、会话失效与屏保计时。 -- [ ] 默认关闭；开发阶段先用 Mosaico 专属编译开关。交付前提供用户可操作的开启/关闭入口，并复用现有设置保存/取消语义。 +- [ ] 默认关闭；开发阶段先用 Mosaico 专属编译开关。交付前提供用户可操作的开启/关闭入口：设置页独立 `CAMERA` 开关，与显示设置共用保存/取消；已保存为打开时顶栏显示摄像头标识。 - [ ] 无摄像头、采集超时、推理失败、内存不足时禁用视觉输入，保留触摸/按键/BLE，不以 fatal error 重启整个面板。初版不承诺热插拔自动恢复，可在重新启用时重试。 - [ ] OTA 和配对期间暂停视觉工作，设置期间禁止动作；恢复后清空轨迹并重新确认手离开。处理进行中的 DMA/推理所有权后再停止。 @@ -132,7 +134,7 @@ CameraBoard / esp_video | 范围 | 检查与建议门槛 | | --- | --- | -| 纯逻辑 | 在现有 Mosaico logic 测试入口增加少量有意义的轨迹用例：左右、抖动/纵向、漏检/目标跳变、持续停留、冷却/离开、旋转/上下文失效；不模拟整套相机驱动 | +| 纯逻辑 | 在现有 Mosaico logic 测试入口增加少量有意义的轨迹用例：上下左右、抖动/斜向、漏检/目标跳变、持续停留、冷却/离开、旋转/上下文失效；不模拟整套相机驱动 | | 屏保 | 若共享控制器有改动，扩展现有 panel_state 测试：首挥只唤醒、恢复原页、重复进入幂等、无效检测不延长超时；保留现有触摸/按键/远程用例 | | 编译 | 手势开/关均构建；共享状态若变化，构建受影响目标。检查锁文件、镜像大小、分区和生成配置；测试镜像构建与测试执行分别记录 | | 方向/识别 | 记录实际距离、光线与板方向；每朝向每方向 20 次，有效操作成功率目标 ≥90%，零反向，零单轮多次触发 | @@ -158,7 +160,7 @@ eim run "idf.py -C firmware/test_apps/panel_state build" v6.1 - 固定 registry 依赖和模型包；左槽 OV3640 采集适配、双缓冲、UYVY 转 RGB888 和四向映射；来源见 [CAMERA_UPSTREAM.md](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)。 - 单手轨迹、OK 500 ms 停留、离开后再触发、有界事件与上下文失效；主循环实现翻页、只唤醒、幂等进入时钟及受保护状态屏蔽。 -- 本地设置开关默认关闭，保存/取消，停用释放采集与模型，错误反馈与手动重试。ESP-DL 内部部分分配使用断言；资源预检查降低风险，但不保证所有运行时内存不足均能无重启降级。 +- 设置页独立 `CAMERA` 开关默认关闭，与显示设置共用保存/取消；顶栏摄像头标识区分采集中、暂停与失败，不显示文字提示；停用释放采集与模型，手动重试。ESP-DL 内部部分分配使用断言；资源预检查降低风险，但不保证所有运行时内存不足均能无重启降级。 - 7 MiB 双 OTA 分区、Folder Push 上限和 Bridge 容量配置；无共享模型分区。 - 手势开/关固件、Mosaico logic、共享 panel_state 测试镜像以及 AMOLED 2.16 固件构建通过。31 个手势/屏保主机 Unity 测试通过 ASan/UBSan；Bridge 620 项测试通过，7 项跳过。测试镜像尚未在设备运行。 diff --git a/docs/validation/mosaico-gestures.md b/docs/validation/mosaico-gestures.md index 87a1342..6c320e2 100644 --- a/docs/validation/mosaico-gestures.md +++ b/docs/validation/mosaico-gestures.md @@ -1,30 +1,14 @@ # Mosaico 手势固件：首次烧录与验收 -本功能在设备本地运行：左挥下一页、右挥上一页；时钟中首次挥手只恢复原页面；👌 OK 稳定停留约 500 ms 进入时钟。一轮操作最多触发一次，需要手离开后再操作。触摸和 AI 键继续可用。 +本功能在设备本地运行：左挥或上挥上一页、右挥或下挥下一页；时钟中首次挥手只恢复原页面；👌 OK 稳定停留约 500 ms 进入时钟。一轮操作最多触发一次，需要手离开后再操作。触摸和 AI 键继续可用。 这是供实机验收的实现。软件测试与固件构建不能证明相机成像、方向、识别率、功耗或长期稳定性；这些仍需在目标设备验证。 -## 首次安装的分区变化 +## 当前分区与构建 -两款模型与固件共同超过原有 4 MiB 应用槽。新布局使用 7 MiB 双 OTA 槽，模型内嵌，支持代码与模型一起更新和回滚。 +16 MiB Flash 使用两个 7 MiB 应用槽，模型与代码共同更新。NVS 位于 `0x9000`，ota_0 位于 `0x10000`，otadata 位于 `0x810000`，ota_1 位于 `0x820000`。Folder Push 限额为 7 MiB + 512 字节，目标注册表和 Bridge 下载上限同步匹配。 -| 分区 | 原布局 | 新布局 | -| --- | --- | --- | -| NVS | `0x9000` / `0x6000` | 保持 | -| PHY | `0xf000` / `0x1000` | 保持 | -| ota_0 | `0x10000` / `0x400000` | `0x10000` / `0x700000` | -| otadata | `0x810000` / `0x2000` | 保持 | -| ota_1 | `0x410000` / `0x400000` | `0x820000` / `0x700000` | - -第二槽结束于 `0xf20000`，在 16 MiB Flash 内；`0x710000` 到 `0x810000` 的空隙保留现有 otadata 地址。Folder Push 限额同步为 7 MiB + 512 字节，目标注册表和 Bridge 下载上限同步更新。 - -**旧 4 MiB 分区不能通过应用 OTA 安装本镜像。首次必须通过有线写入新分区表、应用和初始 OTA 元数据。**设备原有容量检查会拒绝超大镜像；不要绕过检查。现有已安装 Bridge 若仍使用 4 MiB 下载上限，也不能用于传送新镜像。 - -优先使用构建生成的分段烧录参数，保持 NVS 扇区不被覆盖。合并的 full 镜像包含分区间填充，整段写入可能覆盖 NVS，不能用它声称保留配对/设置。本次实现不自动烧录或擦除任何设备。 - -## 从源码准备与烧录 - -在仓库根目录： +在仓库根目录运行： ```sh eim --version @@ -32,36 +16,34 @@ eim list eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 ``` -确认生成配置包含 `CONFIG_MOSAICO_GESTURE_INPUT=y`、`CONFIG_CAMERA_OV3640=y`、640×480 UYVY 模式，`CONFIG_ESP_DESKTOP_BUDDY_FOLDER_PUSH_MAX_TRANSFER_BYTES=7340544`，以及 `QF-Mosaico-` 广播前缀。旧生成配置可能保留不同前缀，不能只检查 defaults。构建产物中的 `flash_args` 是本次镜像的地址来源。 +确认日志含 `Project build complete`，并检查生成配置启用 CLIB 分配器、OV3640 与 SC101IOT。SC101IOT 使用完整 1280×720 UYVY，驱动标称 15 fps；OV3640 使用 640×480 UYVY。 -设备接入左侧 OV3640 CameraBoard。先在设备设置中关闭手势，或断电安装相机板；使用真实串口替换下述 `PORT`。手动进入下载模式后，在 **build 目录**运行： +## 有线烧录注意事项 -```sh -python -m esptool --chip esp32s31 --port PORT --baud 460800 \ - --before no-reset --after no-reset --no-stub write-flash @flash_args -``` +相机板安装在左槽，安装前断电。在已授权的有线烧录中，按住 BOOT 再开机进入 ROM 下载模式。Mosaico 实测整段擦写大应用可能断连；不要直接使用 `idf.py flash` 或将未拆分的大应用交给 `@flash_args`。 -该命令需要已启用的 ESP-IDF v6.1 Python 环境。等待逐段校验成功后手动重启。它写入 bootloader、分区表、ota_0 和 `ota_data_initial.bin`；不包含 NVS，也不执行 `erase-flash`。原有第二槽内容被新的大应用覆盖，首次迁移后不能依赖旧第二槽自动回滚。 +将应用按 1 MiB 拆分，地址从目标应用槽起点逐段递增，使用 ESP-IDF v6.1 环境中的 esptool，以 `--chip esp32s31 --no-stub`、460800 波特率、DIO/80 MHz/16 MB 参数写入，并指定 `--after no-reset`。烧录前校验分段拼接内容与构建应用一致，烧录后确认每段设备哈希校验成功，再通过 POWER 关机重启。 -如需恢复旧版，先准备旧版的完整、匹配的分段文件（旧分区表、旧应用、bootloader、初始 OTA 元数据），再次进入下载模式写入它们。不要单独写回旧分区表，也不要省略重置 OTA 元数据；旧配对和显示设置所在 NVS 地址未变。未来在新布局内进行正常 A/B OTA 时，无需重复迁移分区。 +仅更新当前应用时，不写入 NVS、分区表或初始 OTA 元数据。需要完整安装时，使用构建产物匹配的 bootloader、分区表及初始化数据；应用仍分段写入。不要通过整片擦除或包含填充区域的合并镜像声称保留配对与设置。 ## 启用与操作 -1. 长按现有显示设置入口，选择 `HAND GESTURES / ON`，点击 `SAVE`。默认关闭；取消不会启用摄像头。 -2. 等待 `CAMERA STARTING` 消失，先把手移出画面至少约半秒，再开始动作。 -3. 正常页左右挥动；一次只翻一页。屏保中第一次挥动只恢复原页。 +1. 长按现有显示设置入口，打开 `CAMERA` 开关并点击 `SAVE`。默认关闭；`CANCEL` 不会启用摄像头。 +2. 顶栏摄像头标识变为亮色后，先把手移出画面至少约半秒，再开始动作。 +3. 正常页上下左右挥动；上/左为上一页，下/右为下一页，一次只翻一页。屏保中第一次挥动只恢复原页。 4. 正常页做 👌 OK 并短暂停留；进度条完成才进入时钟。进入后先移开手，下一轮挥手再唤醒。 -5. 设置、配对确认和 OTA 期间不接受手势。关闭并保存后释放采集和模型资源。 +5. 设置、配对确认和 OTA 期间不接受手势，标识转暗。关闭 `CAMERA` 开关并保存后释放采集和模型资源，顶栏标识消失。 -`CHECK LEFT CAMERA` 表示采集未能启动或连续失败。检查左槽相机，关闭并保存，再开启并保存可重试。`LOW MEMORY` 表示资源预检查未通过；`RESTART DEVICE` 表示驱动资源未能安全释放，需要重启后再试。 +顶栏标识带斜杠表示采集失败：相机未能启动或连续采集失败、资源预检查未通过，或驱动资源未能安全释放。检查左槽相机后，把 `CAMERA` 关闭并保存、再打开并保存可重试；仍带斜杠时重启设备。具体原因见 UART0 日志。 -摄像头占用 USB Serial/JTAG 的 GPIO33；运行期间使用 UART 获取诊断。日志只报告动作、周期时间、内存和栈余量，不输出图像或用户数据。屏保时摄像头仍工作，因此本功能不是深睡眠唤醒，也未验证续航收益。 +摄像头占用左侧排针上的 USB Serial/JTAG（GPIO33/34）。Type-C 口是 USB OTG，只在 ROM 下载模式下枚举，与摄像头无关；应用日志只从 UART0 输出（右侧排针 H1 的 15 脚 TX0 / GPIO58，或底板背面调试焊盘 TX，115200）。采集失败时每 5 秒打印一次 `capture unavailable: <步骤> (code N)`。板上没有 RST 键：复位用 POWER 键关机再开机，下载模式为关机后按住 BOOT 再开机。日志只报告动作、周期时间、内存和栈余量，不输出图像或用户数据。屏保时摄像头仍工作，因此本功能不是深睡眠唤醒，也未验证续航收益。 ## 实机验收顺序 - 首先确认原有触摸、AI 键、BLE、显示旋转和 OTA 确认界面工作正常。 +- 确认 `CAMERA` 开关保存后重启仍保持，切换开关会启用 `SAVE`、`CANCEL` 会恢复；保存为打开时顶栏标识出现，采集中为亮色，设置页、配对或 OTA 期间为暗色，失败时带斜杠；关闭并保存后标识消失。 - 检查相机能连续采集，记录 `gesture_input` 周期与内存日志；缺相机时面板应继续工作。 -- 在每个屏幕朝向检查左右语义。默认相机安装修正为 180°、镜像开启；若方向或手形识别不正确，使用 `menuconfig → Camera gestures` 校准后重建，不能把主机坐标转换测试当作硬件方向验证。 +- 在每个屏幕朝向检查左右语义。默认先水平镜像再顺时针旋转 90°，与官方参考实现的逆时针 90° 正立画面一致；若方向或手形识别不正确，使用 `menuconfig → Camera gestures` 校准后重建，不能把主机坐标转换测试当作硬件方向验证。 - 分别在三个用量页进入时钟，再挥手，确认回到原页且不继续翻页。保持 OK、自然松手和单帧漏检都不应反复切换。 - 测试设置/配对/OTA 期间持续挥手，退出后不能回放旧动作；随后移开手再做新动作应恢复。 - 记录实际距离、照明和方向；每朝向每方向至少 20 次，并在打字、拿杯等场景观察至少 30 分钟误触。连续运行及反复启停，检查内存是否稳定。 @@ -69,9 +51,60 @@ python -m esptool --chip esp32s31 --port PORT --baud 460800 \ 识别距离、方向标定、模型延迟、功耗、实际摄像头热插拔和 A/B 回滚效果均为硬件验收项；初版只提供关闭/开启重试，不承诺热插拔自动恢复。 -## 本次软件验证 +## 初始实现的软件验证记录 - ESP-IDF v6.1：手势开启/关闭固件均构建成功；Mosaico logic 和共享 panel_state 测试镜像构建成功；受共享屏保接口影响的 AMOLED 2.16 固件构建成功。 -- 主机实际运行 31 个手势/屏保 Unity 测试，ASan/UBSan 通过；测试镜像未在 MCU 上运行。 -- Bridge：620 项测试通过，7 项跳过，包含新 7 MiB 发布清单容量和第三方许可检查。 -- 未进行实机烧录。ESP-DL 内部模型分配包含断言；预检查不能替代内存压力与启停实测。 +- 手势轨迹与屏保 Unity 测试曾在主机实际运行并通过 ASan/UBSan。之后新增的漏检、低置信度、慢帧与位置跳变用例仅随 logic 测试镜像编译，尚未在主机或 MCU 上运行；测试镜像未在 MCU 上运行。 +- `CAMERA` 开关与顶栏标识只经过固件构建，布局与触控需实机确认。 +- Bridge 全部测试通过，包含新 7 MiB 发布清单容量和第三方许可检查。 +- 初始验证时未进行实机烧录；后续烧录与设备验证见下文。ESP-DL 内部模型分配包含断言，预检查不能替代内存压力与启停实测。 + +## 时钟黑屏排查（2026-09-23） + +故障固件使用 LVGL 内置 64 KiB 内存池，未启用扩容。原始 UART 日志中的 ELF 哈希与排查用 ELF 匹配；进入时钟后，`swdraw` 的看门狗现场持续停在 `lv_draw_unit_draw_letter` 的 `LV_ASSERT_MALLOC(draw_buf)`。反汇编确认该地址为自跳转指令，前置分支仅在字形缓冲分配返回 NULL 时进入。因此，已确认直接故障为字形缓冲分配失败后断言死循环；现有日志不能区分内存池总余量不足与碎片化，也没有 main 分支的运行时内存池余量对照。 + +Mosaico 默认配置改为 `CONFIG_LV_USE_CLIB_MALLOC=y`，使界面对象和字形缓冲使用 ESP-IDF 标准堆。已有工作目录还需在生成的 `sdkconfig` 中关闭 `CONFIG_LV_USE_BUILTIN_MALLOC` 并启用 CLIB，重新构建后检查 `build/config/sdkconfig.h`。标准堆是否使用 PSRAM 由 ESP-IDF 的分配策略决定，不能将 CLIB 等同于强制 PSRAM 分配。 + +本轮 ESP-IDF v6.1 构建完成；生成配置启用 CLIB 和调试预览，最终 ELF 的 lv_malloc_core 直接跳转到标准 malloc。通过 UART0 被动读取的修复前设备日志仍停在同一断言地址。修复镜像已按 1 MiB 分段写入应用区，六段均通过设备哈希校验；未写入 NVS、分区表或 OTA 元数据。重启日志的 ELF 哈希与修复产物一致，摄像头和 BLE 均恢复运行。用户确认通过 OK 进入时钟后显示正常；后续 UART 采集日志持续输出，未再出现看门狗报警。日志另捕获到 SwipeLeft/Next 与 SwipeRight/Previous 各一次；这只能证明两个方向均已触发，不能代替触发率验收。 + +后续压力验收：保持调试预览开启，反复通过 OK 进入时钟并唤醒，覆盖分钟切换、触摸、BLE 和摄像头启停；确认画面正常、无 `swdraw` 看门狗，且现有 `internal_free`、`psram_free` 日志无持续下降。CLIB 后端不提供有效的 `lv_mem_monitor` 统计，应使用 ESP-IDF 堆统计。左右挥手的触发率仍待单独验证。 + +## 近距离取景调整 + +SC101IOT 使用完整 1280×720 画面，避免 VGA 模式的中心裁切；等比例缩为 320×180 RGB888，旋转后将实际尺寸传给模型和预览。OV3640 保持 640×480 采集、320×240 RGB888。手势阈值保持不变，以便单独验证取景范围变化。 + +本版应用与 logic 测试镜像均已通过 ESP-IDF v6.1 构建，新增图像比例、旋转、镜像和边缘保留用例仅完成编译，尚未运行。应用已按六段写入设备，全部通过设备哈希校验；未写入 NVS、分区表或 OTA 元数据。重启日志确认运行镜像与产物匹配，并成功采集 1280×720 UYVY；观察窗口内无看门狗或采集失败，记录到一次右挥翻页。稳定采样的平均处理周期约 105～123 ms，PSRAM 空闲约 4.39 MB。用户反馈近距离取景有所改善，但旋转后的竖幅横向空间不足；具体距离、OK 识别及左右挥手触发率尚未验收。检查启动日志包含 `camera UYVY 1280x720`，预览应包含更完整的场景且手形无拉伸；分别在约 40、30、25 cm 检查完整入镜、OK 进入时钟与左右挥手。记录实际识别率、平均周期、帧超时和内存余量，确认反复关闭/开启 CAMERA 与时钟切换正常。25～30 cm 是测试目标，不是已验证的识别距离。 + + +## 空中挥手的页面方向 + +空中手势按导航项顺序选择页面：右挥或下挥为下一页，左挥或上挥为上一页，末尾按现有页面顺序循环。触摸滑动语义保持不变；时钟中首次挥手仍只唤醒。前述日志中的 SwipeRight/Previous 等记录属于调整前版本，不作为当前映射的验收证据。 + +方向调整后的应用与 logic 测试镜像均通过 ESP-IDF v6.1 构建；路由测试尚未运行，方向修正已随四向手势应用烧录，方向一致性待实机确认。 + + +## 上下挥手 + +上下挥手与左右挥手共用单手置信度、150～900 ms 动作窗口、至少三帧、25% 主轴净位移和释放确认。斜向轨迹及往返抖动不应翻页；时钟中任一方向首次挥动只恢复原页。实机需分别验证上挥上一页、下挥下一页，以及一次动作不会重复翻页；新增四向逻辑尚未实机验收。 + +四向手势应用和 logic 测试镜像均通过 ESP-IDF v6.1 构建；扩展后的轨迹、斜向/抖动拒绝及动作路由测试尚未运行。应用已按六段写入设备，全部通过设备哈希校验；仅更新应用分区，未写入 NVS、分区表或 OTA 元数据。烧录结束后设备停留在下载模式，后续日志及用户反馈确认：手离开镜头后，上挥可唤醒时钟；其余方向和连续操作的完整验收仍待完成。 + +## 相机图标预览开关 + +预览默认隐藏，点击顶部相机图标切换显示；隐藏时跳过预览绘制，手势识别继续运行。应用通过 ESP-IDF v6.1 构建，已按六段烧录并全部通过设备哈希校验。仅更新应用分区，重启后已采集到分阶段性能日志，图标点击行为仍待用户确认。本版本包含采集转换、检测、分类和预览反馈的分阶段耗时统计。 + +## 性能基线与后续优化（2026-09-23） + +预览关闭且未执行 OK 分类的三个连续约 10 秒窗口中，每窗口处理 80 帧，约 8 fps。平均采集与转换耗时 18.6 ms，手掌检测（含前后处理）86.8 ms，反馈约 0.006 ms，循环另有固定 20 ms 等待，总帧间隔约 125 ms。`mean_cycle_ms` 不含反馈及固定等待，不能直接取倒数作为识别帧率。该窗口不代表正常页检测到手后执行 OK 分类的完整负载。 + +[官方手掌检测基准](https://github.com/espressif/esp-dl/tree/master/models/hand_detect)给出同模型 S3 合计 132.8 ms、P4 合计 50.9 ms，未列 S31。[官方 S31 案例](https://esp32-s31.espressif.com/en/official-projects/6)中同系列 ESPDet-Pico 猫检测合计 94.9 ms，仅用于量级参考，不能视为相同模型的直接对比。当前 CPU 为 320 MHz，PSRAM 为 250 MHz。 + +后续按以下顺序评估，尚未实现，收益不得作为已验证结果： + +1. 缩短固定 20 ms 等待，并检查 BLE、显示、看门狗和任务公平性。其他耗时不变时，理论上可接近 9.5 fps。 +2. 将采集等待与图像转换继续拆分计时，再对照官方相机实现评估硬件缩放、旋转及转换。保留完整传感器视野，不通过中心裁切换取速度。 +3. 测量正常页有手时的 OK 分类成本；评估移动时优先跟踪、稳定后再分类。时钟中已跳过 OK 分类，不能重复计算这项收益。 +4. 若上述收益不足，再评估更轻模型或检测与轻量跟踪交替运行；验证快速运动、遮挡、多手及重新捕获，避免累计漂移。 +5. 保留至少三帧确认，结合实际帧率评估 150 ms 最短动作时间、位移及位置跳变阈值。分别测量左右与上下挥手的成功率、误触率及唤醒延迟。 + +近距离优化需区分光学视野与识别条件：当前已使用 SC101IOT 完整画面；软件缩小预览不能扩大视角。可评估较小挥手幅度及大手掌框的过滤阈值，部分手掌入镜的识别可靠性需实测。预览开关的性能对比应采用相同手势、页面与照明条件。 diff --git a/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md b/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md index 5026daf..de14b6c 100644 --- a/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md +++ b/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md @@ -20,7 +20,7 @@ commits below and [dependencies.lock](dependencies.lock), not the latest branch. - Read the [interaction design and implementation record](../../../docs/design/mosaico-gesture-input.md) for swipe direction, wake-only behavior, OK hold and protected states. - Start in [gesture_camera.cpp](main/gesture_camera.cpp) for capture/pin ownership, [gesture_image.cpp](main/gesture_image.cpp) for image orientation, [gesture_tracker.cpp](main/gesture_tracker.cpp) for temporal recognition, and [gesture_input.cpp](main/gesture_input.cpp) for the worker lifecycle. [app_main.cpp](main/app_main.cpp) owns action routing and UI state. - Follow the [installation and hardware acceptance guide](../../../docs/validation/mosaico-gestures.md). Software builds/tests passed at the initial implementation; camera orientation, actual recognition, UI layout, resource use and power remain hardware acceptance items. -- Preserve the 7 MiB dual-slot migration contract, default-off setting, local-only image processing and existing BSP display fixes. Recheck the current tree and lockfile before changing dependencies; do not edit generated `managed_components` as the source of truth. +- Preserve the 7 MiB dual-slot layout, default-off capture setting, local-only image processing and existing BSP display fixes. Recheck the current tree and lockfile before changing dependencies; do not edit generated `managed_components` as the source of truth. ## Locked component versions @@ -29,7 +29,7 @@ The target uses registry components pinned by `dependencies.lock`: | Component | Version | Role | | --- | --- | --- | | espressif/esp_video | 2.5.0 | DVP/V4L2 capture and buffer lifetime | -| espressif/esp_cam_sensor | 2.6.0 | OV3640 sensor configuration | +| espressif/esp_cam_sensor | 2.6.0 | OV3640 and SC101IOT sensor configuration | | espressif/esp-dl | 3.3.11 | S31 inference and preprocessing | | espressif/hand_detect | 0.2.0 | ESPDet-Pico 224×224 hand detection | | espressif/hand_gesture_recognition | 0.2.0 | MobileNetV2 128×128 classification; `ok` command | @@ -54,11 +54,38 @@ Serial/JTAG. GPIO34 is the active-low illuminator and remains high. UART logging is retained. Start/stop and frame ownership belong to the vision worker; the application task exchanges context and bounded events only. -Camera mounting rotation and mirror defaults remain subject to physical -calibration. `CONFIG_MOSAICO_CAMERA_ROTATION` applies the mounting correction; -the inverse of the actual applied display rotation follows it. A context change -discards any unfinished gesture. The default correction is 180 degrees, with -horizontal mirroring enabled. +Start-up timing follows `boards/ESP32_S31_MOSAICO/camera.c` in +[esp-vision](https://github.com/espressif/esp-vision) and `mosaico_module_camera` +in [esp-mosaico-bsp](https://github.com/esp-mosaico/esp-mosaico-bsp): 20 ms of +stabilization after the slot is claimed, a 500 ms frame timeout, up to three +start attempts 100 ms apart, and one discarded frame per buffer after stream +start. Frames flagged with `V4L2_BUF_FLAG_ERROR` are requeued unconverted. +SCCB runs at 100 kHz on the BSP subboard I2C1 bus, matching +`mosaico_module_camera`; that bus has only internal pull-ups. CameraBoards are +fitted with either an OV3640 (SCCB 0x3c) or an SC101IOT (0x68); both are built +with auto-detection, as in `mosaico_module_camera`. OV3640 uses 640x480 UYVY; +SC101IOT uses its full 1280x720 UYVY frame at the driver's nominal 15 fps. +The SC101IOT VGA register table crops the center of the 1280x720 array, so +720p retains more of the scene for close-range gestures. Format negotiation +follows `mosaico_module_camera`: read the configured sensor size with +`VIDIOC_G_FMT`, request UYVY with `VIDIOC_S_FMT`, and verify the returned size. +No sensor register table or board timing is modified. + +The complete frame is fitted within 320x240 while preserving its aspect ratio: +SC101IOT produces 320x180 RGB888 before rotation, and OV3640 produces 320x240. +The model and debug preview receive the actual rotated dimensions. The two +720p UYVY capture buffers require about 3.52 MiB; the startup precheck requires +at least a 9 MiB contiguous free PSRAM block for capture, model arenas and UI load. +The nominal sensor frame rate and close-range recognition require hardware +verification on the CameraBoard's external oscillator. + +`CONFIG_MOSAICO_CAMERA_ROTATION` applies the mounting correction after the +horizontal mirror; the inverse of the actual applied display rotation follows +it. A context change discards any unfinished gesture. Both reference pipelines +rotate the raw frame 90 degrees counter-clockwise (`PPA_SRM_ROTATION_ANGLE_90`) +without mirroring to make it model-upright. Mirroring first keeps swipe +directions in the user's view, so the equivalent default is 90 degrees +clockwise with mirroring enabled. Package copyright and license sources are recorded in [THIRD_PARTY_LICENSES.md](../../../THIRD_PARTY_LICENSES.md). diff --git a/firmware/targets/esp_mosaico/README.md b/firmware/targets/esp_mosaico/README.md index 9dec2df..5c51d98 100644 --- a/firmware/targets/esp_mosaico/README.md +++ b/firmware/targets/esp_mosaico/README.md @@ -40,7 +40,7 @@ eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 后续开发入口：[参考仓库、固定版本与代码导航](CAMERA_UPSTREAM.md#repository-reference-map)；[交互设计与实施记录](../../../docs/design/mosaico-gesture-input.md)。 -左槽 OV3640 CameraBoard 可选开启本地手势：左挥下一页、右挥上一页，时钟中首挥只恢复原页；👌 OK 稳定保持约 500 ms 进入时钟。设置中的 `HAND GESTURES` 默认关闭，需保存后启用。图像仅在本机处理。 +左槽 CameraBoard（OV3640 或 SC101IOT，自动识别）可选开启本地手势：左挥或上挥上一页、右挥或下挥下一页，时钟中首挥只恢复原页；👌 OK 稳定保持约 500 ms 进入时钟。设置中的 `CAMERA` 开关默认关闭，切换后需 `SAVE`；开关打开时顶栏显示摄像头标识。Type-C 口是 USB OTG，只在 ROM 下载模式下枚举，应用日志只从 UART0（右侧排针 H1 的 15 脚 TX0 / GPIO58，或底板背面调试焊盘 TX）输出。图像仅在本机处理。 模型与代码内嵌同一应用，首次从旧 4 MiB 分区升级必须有线迁移。保留设置时使用分段烧录，不能用填充后的完整镜像替代。依赖来源见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)，安装、恢复与实机验证见[手势验收指南](../../../docs/validation/mosaico-gestures.md)。当前仅通过软件测试和构建；相机方向、识别率、性能、功耗和提示布局仍待设备验收。 @@ -80,3 +80,5 @@ eim run "idf.py -C firmware/targets/esp_mosaico/test_apps/logic build" v6.1 1. 实机验证触控方向、AI 键，以及 OTA 断电恢复与回滚； 2. 按实测结果校准 BQ27220 的外部供电判据。 + +点击顶部相机图标可显示或隐藏实时预览；预览默认隐藏，重启后恢复隐藏。关闭预览不影响手势识别，摄像头采集仍由设置页 CAMERA 开关控制。时钟、设置及受保护界面不显示预览。 diff --git a/firmware/targets/esp_mosaico/main/CMakeLists.txt b/firmware/targets/esp_mosaico/main/CMakeLists.txt index ae8aa2b..c4494e9 100644 --- a/firmware/targets/esp_mosaico/main/CMakeLists.txt +++ b/firmware/targets/esp_mosaico/main/CMakeLists.txt @@ -5,7 +5,7 @@ idf_component_register( "local_clock.cpp" "orientation.cpp" "power_monitor.cpp" - "panel_settings.cpp" + "camera_setting.cpp" "gesture_input.cpp" "gesture_tracker.cpp" INCLUDE_DIRS "." diff --git a/firmware/targets/esp_mosaico/main/Kconfig.projbuild b/firmware/targets/esp_mosaico/main/Kconfig.projbuild index 4763d01..569090a 100644 --- a/firmware/targets/esp_mosaico/main/Kconfig.projbuild +++ b/firmware/targets/esp_mosaico/main/Kconfig.projbuild @@ -94,18 +94,21 @@ config MOSAICO_GESTURE_INPUT bool "Build local CameraBoard gesture input" default y help - Include OV3640 capture and ESP-DL models. Runtime capture remains off + Include CameraBoard capture (OV3640 or SC101IOT) and ESP-DL models. Runtime capture remains off until enabled and saved in the display settings. The camera occupies the left slot and GPIO33, so USB Serial/JTAG cannot be used concurrently. config MOSAICO_CAMERA_ROTATION int "Camera mounting correction (0, 90, 180 or 270 degrees clockwise)" range 0 270 - default 180 + default 90 depends on MOSAICO_GESTURE_INPUT help - Rotates the raw image into panel rotation-0 coordinates. The current - display rotation is then inverted. Calibrate this on the physical board. + Rotates the mirrored raw image into panel rotation-0 coordinates. The + current display rotation is then inverted. The reference pipelines + (mosaico_module_camera, esp-vision) rotate the raw frame 90 degrees + counter-clockwise without mirroring to make it model-upright; with + mirroring applied first, the same upright view needs 90 clockwise. config MOSAICO_CAMERA_MIRROR bool "Mirror the camera horizontally before rotation" @@ -114,6 +117,15 @@ config MOSAICO_CAMERA_MIRROR help Mirror the front-facing camera into the user's screen coordinates. +config MOSAICO_GESTURE_DEBUG_PREVIEW + bool "Enable camera preview toggled by the status icon" + default y + depends on MOSAICO_GESTURE_INPUT + help + The status-bar camera icon toggles a preview of the model input, + detection box and confidence scores. The preview starts hidden and + does not change whether gesture recognition is enabled. + endmenu menu "Bluetooth" diff --git a/firmware/targets/esp_mosaico/main/app_main.cpp b/firmware/targets/esp_mosaico/main/app_main.cpp index a7b55d6..1ca86d9 100644 --- a/firmware/targets/esp_mosaico/main/app_main.cpp +++ b/firmware/targets/esp_mosaico/main/app_main.cpp @@ -5,7 +5,7 @@ #include "display_ui.hpp" #include "local_clock.hpp" #include "gesture_input.hpp" -#include "panel_settings.hpp" +#include "camera_setting.hpp" #include "usage_panel_state/panel_presentation.hpp" #include "power_monitor.hpp" #include "usage_panel_state/render_schedule.hpp" @@ -194,13 +194,24 @@ extern "C" void app_main(void) } const bool ai_button_ready = init_ai_button(); - PanelSettings saved_settings = load_panel_settings({ + DisplaySettings saved_settings = load_display_settings({ CONFIG_MOSAICO_USAGE_PANEL_DISPLAY_BRIGHTNESS_PERCENT, CONFIG_MOSAICO_USAGE_PANEL_SCREENSAVER_TIMEOUT_SECONDS}); - PanelSettings settings = saved_settings; + DisplaySettings settings = saved_settings; bool settings_open = false; + // camera_enabled is persisted and drives capture; camera_setting is the + // settings-page preview, committed by SAVE together with the display values. + bool camera_enabled = load_camera_enabled(); + bool camera_setting = camera_enabled; +#if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW + gestures.set_preview_sink( + [](void* context, const uint8_t* rgb, unsigned width, unsigned height, const HandObservation& hand) { + static_cast(context)->show_camera_preview(rgb, width, height, hand); + }, + &ui); +#endif if (!gestures.begin()) ESP_LOGW(TAG, "gesture worker unavailable"); - ui.apply_settings(settings, false); + ui.apply_settings(settings, camera_setting, false); ScreenRotation desired_rotation = initial_rotation; // ui.begin() brought the panel up at kFixedRotation. Anything the startup @@ -295,22 +306,21 @@ extern "C" void app_main(void) const bool settings_blocked = ble_link.has_passkey || ota_view.phase != OtaPhase::Idle; if (settings_open && settings_blocked) { settings = saved_settings; + camera_setting = camera_enabled; settings_open = false; - ui.apply_settings(settings, false); + ui.apply_settings(settings, camera_setting, false); screensaver.reset(now_ms, page); dirty = true; } - if (ui.take_gesture_toggle() && settings_open && !settings_blocked) { - settings.gestures = !settings.gestures; - ui.apply_settings(settings, true, false, settings_differ(settings, saved_settings)); - dirty = true; - } - if (action != SettingsAction::None && !settings_blocked) { + const bool camera_toggled = ui.take_camera_toggle(); + if ((action != SettingsAction::None || camera_toggled) && !settings_blocked) { bool save_error = false; - const PanelSettings before = settings; + const DisplaySettings before = settings; + const bool camera_before = camera_setting; const bool was_open = settings_open; if (action == SettingsAction::Open) settings_open = true; else if (settings_open) { + if (camera_toggled) camera_setting = !camera_setting; if (action == SettingsAction::Dimmer) settings.brightness = settings.brightness <= 10 ? 1 : settings.brightness - 10; else if (action == SettingsAction::Brighter) @@ -318,18 +328,26 @@ extern "C" void app_main(void) else if (action >= SettingsAction::TimeoutOff && action <= SettingsAction::Timeout30) select_clock_timeout(action, settings.clock_timeout_seconds); else if (action == SettingsAction::Save) { - const esp_err_t error = save_panel_settings(settings); + esp_err_t error = save_display_settings(settings); + if (error == ESP_OK) error = save_camera_enabled(camera_setting); save_error = error != ESP_OK; - if (!save_error) { saved_settings = settings; settings_open = false; } - else ESP_LOGW(TAG, "display settings save failed: %s", esp_err_to_name(error)); + if (!save_error) { + saved_settings = settings; + camera_enabled = camera_setting; + settings_open = false; + } else ESP_LOGW(TAG, "settings save failed: %s", esp_err_to_name(error)); } else if (action == SettingsAction::Cancel) { settings = saved_settings; + camera_setting = camera_enabled; settings_open = false; } } - if (!ui.apply_settings(settings, settings_open, save_error, - settings_differ(settings, saved_settings))) { + if (!ui.apply_settings(settings, camera_setting, settings_open, save_error, + settings.brightness != saved_settings.brightness || + settings.clock_timeout_seconds != saved_settings.clock_timeout_seconds || + camera_setting != camera_enabled)) { settings = before; + camera_setting = camera_before; settings_open = was_open; ESP_LOGW(TAG, "display settings preview failed"); } @@ -361,15 +379,17 @@ extern "C" void app_main(void) ui.request_page(page); screensaver.reset(now_ms, page); settings = saved_settings; + camera_setting = camera_enabled; settings_open = false; - ui.apply_settings(settings, false); + ui.apply_settings(settings, camera_setting, false); status_page.store(page, std::memory_order_relaxed); } } else if (event.type == AppEventType::ScreenToggle) { gestures.invalidate(); settings = saved_settings; + camera_setting = camera_enabled; settings_open = false; - ui.apply_settings(settings, false); + ui.apply_settings(settings, camera_setting, false); const bool was_asleep = screensaver.view(now_ms).active; screensaver.note_remote_toggle(now_ms, page); if (was_asleep) { @@ -472,7 +492,7 @@ extern "C" void app_main(void) ota_view.phase != OtaPhase::Idle || show_ota_error || settings_open); const bool gesture_allowed = !settings_open && !ble_link.has_passkey && ota_view.phase == OtaPhase::Idle && !show_ota_error && desired_rotation == applied_rotation; - gestures.configure(saved_settings.gestures, gesture_allowed, + gestures.configure(camera_enabled, gesture_allowed, screensaver.view(now_ms).active, page, applied_rotation); GestureEvent gesture{}; if (gestures.take(gesture, esp_timer_get_time() / 1000)) { @@ -531,13 +551,15 @@ extern "C" void app_main(void) } const auto gesture_status = gestures.status(); const auto gesture_progress = gestures.progress(); - const bool gesture_feedback_visible = saved_settings.gestures && gesture_allowed && !screensaver_view.active; + const bool gesture_feedback_visible = camera_enabled && gesture_allowed && !screensaver_view.active; + static bool shown_camera_enabled = false; static GestureStatus shown_gesture_status = GestureStatus::Off; static uint8_t shown_gesture_progress = 0; static bool shown_gesture_feedback = false; - if (shown_gesture_status != gesture_status || shown_gesture_progress != gesture_progress || - shown_gesture_feedback != gesture_feedback_visible) { - ui.show_gesture_status(gesture_status, gesture_progress, gesture_feedback_visible); + if ((shown_camera_enabled != camera_enabled || shown_gesture_status != gesture_status || + shown_gesture_progress != gesture_progress || shown_gesture_feedback != gesture_feedback_visible) && + ui.show_camera_state(camera_enabled, gesture_status, gesture_progress, gesture_feedback_visible)) { + shown_camera_enabled = camera_enabled; shown_gesture_status = gesture_status; shown_gesture_progress = gesture_progress; shown_gesture_feedback = gesture_feedback_visible; diff --git a/firmware/targets/esp_mosaico/main/camera_setting.cpp b/firmware/targets/esp_mosaico/main/camera_setting.cpp new file mode 100644 index 0000000..4781f4d --- /dev/null +++ b/firmware/targets/esp_mosaico/main/camera_setting.cpp @@ -0,0 +1,31 @@ +#include "camera_setting.hpp" +#include "nvs.h" + +namespace usage_panel::mosaico { +namespace { +constexpr char kNamespace[] = "camera"; +constexpr char kEnabledKey[] = "enabled"; +} + +bool load_camera_enabled() +{ + nvs_handle_t handle; + if (nvs_open(kNamespace, NVS_READONLY, &handle) != ESP_OK) return false; + uint8_t enabled = 0; + const bool stored = nvs_get_u8(handle, kEnabledKey, &enabled) == ESP_OK && enabled <= 1; + nvs_close(handle); + return stored && enabled; +} + +esp_err_t save_camera_enabled(bool enabled) +{ + nvs_handle_t handle; + esp_err_t error = nvs_open(kNamespace, NVS_READWRITE, &handle); + if (error != ESP_OK) return error; + error = nvs_set_u8(handle, kEnabledKey, enabled ? 1 : 0); + if (error == ESP_OK) error = nvs_commit(handle); + nvs_close(handle); + return error; +} + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/camera_setting.hpp b/firmware/targets/esp_mosaico/main/camera_setting.hpp new file mode 100644 index 0000000..cd530f1 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/camera_setting.hpp @@ -0,0 +1,12 @@ +#pragma once + +#include "esp_err.h" + +namespace usage_panel::mosaico { + +/** Returns the persisted camera switch; absent or invalid storage reads as off. */ +bool load_camera_enabled(); +/** Persists the camera switch; called by settings SAVE together with the display values. */ +esp_err_t save_camera_enabled(bool enabled); + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/display_ui.cpp b/firmware/targets/esp_mosaico/main/display_ui.cpp index 7c3e3be..e451b7f 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.cpp +++ b/firmware/targets/esp_mosaico/main/display_ui.cpp @@ -4,6 +4,7 @@ #include #include "bsp/esp_mosaico.h" +#include "esp_heap_caps.h" #include "esp_log.h" #include "usage_fonts/usage_fonts.h" #include "esp_timer.h" @@ -247,6 +248,19 @@ void style_panel(lv_obj_t* panel, uint32_t background = COLOR_PANEL, lv_obj_remove_flag(panel, LV_OBJ_FLAG_SCROLLABLE); } +/** Creates an unstyled, non-interactive primitive for icons drawn from shapes. */ +lv_obj_t* make_shape(lv_obj_t* parent, int x, int y, int width, int height, int radius) +{ + lv_obj_t* shape = lv_obj_create(parent); + lv_obj_remove_style_all(shape); + lv_obj_set_pos(shape, x, y); + lv_obj_set_size(shape, width, height); + lv_obj_set_style_radius(shape, radius, 0); + lv_obj_remove_flag(shape, LV_OBJ_FLAG_CLICKABLE); + lv_obj_remove_flag(shape, LV_OBJ_FLAG_SCROLLABLE); + return shape; +} + lv_obj_t* make_label(lv_obj_t* parent, const char* text, int x, int y, const lv_font_t* font, uint32_t color = COLOR_TEXT) { @@ -357,6 +371,8 @@ bool DisplayUi::begin(ScreenRotation initial_rotation) build_provider_page(screen, Provider::Codex); build_provider_page(screen, Provider::Claude); build_navigation(screen); + // Below the screensaver and overlays, which raise themselves when shown. + build_camera_preview(screen); build_screensaver(screen); build_settings(screen); build_pairing_overlay(screen); @@ -468,6 +484,38 @@ void DisplayUi::build_header(lv_obj_t* screen) lv_obj_set_style_pad_all(terminal, 0, 0); lv_obj_remove_flag(terminal, LV_OBJ_FLAG_SCROLLABLE); +#if CONFIG_MOSAICO_GESTURE_INPUT + // Solid camera silhouette left of the connection status, vertically centred + // on its dot. The header fonts carry no camera glyph, so it is drawn from + // primitives: body and viewfinder in the state colour, a background-coloured + // lens ring and a state-coloured lens centre. + static const lv_point_precise_t kSlash[] = {{3, 2}, {23, 26}}; + camera_indicator_ = make_shape(screen, 275, 13, 26, 28, 0); +#if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW + lv_obj_add_flag(camera_indicator_, LV_OBJ_FLAG_CLICKABLE); + lv_obj_set_ext_click_area(camera_indicator_, 8); + lv_obj_add_event_cb(camera_indicator_, camera_preview_toggle_event, LV_EVENT_CLICKED, this); +#endif + camera_indicator_parts_[0] = make_shape(camera_indicator_, 6, 3, 8, 4, 1); + camera_indicator_parts_[1] = make_shape(camera_indicator_, 0, 6, 26, 16, 4); + camera_indicator_lens_ = make_shape(camera_indicator_, 8, 9, 10, 10, 5); + camera_indicator_parts_[2] = make_shape(camera_indicator_, 11, 12, 4, 4, 2); + lv_obj_set_style_bg_color(camera_indicator_lens_, lv_color_hex(COLOR_BG), 0); + for (lv_obj_t* part : {camera_indicator_parts_[0], camera_indicator_parts_[1], + camera_indicator_lens_, camera_indicator_parts_[2]}) + lv_obj_set_style_bg_opa(part, LV_OPA_COVER, 0); + for (size_t i = 0; i < 2; ++i) { + lv_obj_t* slash = lv_line_create(camera_indicator_); + camera_indicator_slash_[i] = slash; + lv_line_set_points(slash, kSlash, 2); + lv_obj_set_style_line_width(slash, i == 0 ? 6 : 2, 0); + lv_obj_set_style_line_color(slash, lv_color_hex(i == 0 ? COLOR_BG : COLOR_RESET), 0); + lv_obj_set_style_line_rounded(slash, true, 0); + lv_obj_remove_flag(slash, LV_OBJ_FLAG_CLICKABLE); + } + lv_obj_add_flag(camera_indicator_, LV_OBJ_FLAG_HIDDEN); +#endif + lv_obj_t* divider = lv_obj_create(screen); lv_obj_set_pos(divider, MARGIN, HEADER_H); lv_obj_set_size(divider, CARD_W, 1); @@ -817,18 +865,33 @@ void DisplayUi::build_settings(lv_obj_t* screen) lv_obj_center(label); } #if CONFIG_MOSAICO_GESTURE_INPUT - gesture_toggle_button_ = lv_button_create(settings_panel_); - lv_obj_remove_style_all(gesture_toggle_button_); - lv_obj_set_pos(gesture_toggle_button_, MARGIN, 270); - lv_obj_set_size(gesture_toggle_button_, CARD_W, 44); - style_panel(gesture_toggle_button_); - lv_obj_set_style_bg_color(gesture_toggle_button_, lv_color_hex(COLOR_TRACK), LV_STATE_PRESSED); - gesture_toggle_label_ = make_label(gesture_toggle_button_, "HAND GESTURES / OFF", 12, 12, - &montserrat_semibold_13, COLOR_TEXT); - lv_obj_add_event_cb(gesture_toggle_button_, gesture_toggle_event, LV_EVENT_CLICKED, this); - gesture_status_label_ = make_label(screen, "", MARGIN, HEADER_H + 8, - &montserrat_semibold_13, COLOR_MUTED); - lv_obj_add_flag(gesture_status_label_, LV_OBJ_FLAG_HIDDEN); + divider = lv_obj_create(settings_panel_); + lv_obj_remove_style_all(divider); + lv_obj_set_pos(divider, MARGIN, 264); + lv_obj_set_size(divider, CARD_W, 1); + lv_obj_set_style_bg_color(divider, lv_color_hex(COLOR_LINE), 0); + lv_obj_set_style_bg_opa(divider, LV_OPA_COVER, 0); + title = make_label(settings_panel_, "CAMERA", MARGIN, 276, &lv_font_montserrat_16, COLOR_MUTED); + lv_obj_set_style_text_letter_space(title, 1, 0); + make_label(settings_panel_, "HAND GESTURES / ON DEVICE", MARGIN, 298, + &lv_font_montserrat_14, COLOR_MUTED); + // Not checkable: app_main owns the previewed state and draws it through + // apply_settings, so a failed preview can roll back like the other controls. + camera_switch_track_ = lv_button_create(settings_panel_); + lv_obj_remove_style_all(camera_switch_track_); + lv_obj_set_pos(camera_switch_track_, 376, 272); + lv_obj_set_size(camera_switch_track_, 80, 44); + style_panel(camera_switch_track_, COLOR_PANEL, 22); + lv_obj_set_style_bg_color(camera_switch_track_, lv_color_hex(COLOR_TRACK), LV_STATE_PRESSED); + lv_obj_set_style_border_color(camera_switch_track_, lv_color_hex(COLOR_MUTED), LV_STATE_PRESSED); + lv_obj_set_style_bg_color(camera_switch_track_, lv_color_hex(COLOR_TEXT), LV_STATE_CHECKED); + lv_obj_set_style_border_color(camera_switch_track_, lv_color_hex(COLOR_TEXT), LV_STATE_CHECKED); + lv_obj_set_style_bg_color(camera_switch_track_, lv_color_hex(COLOR_RESET), LV_STATE_CHECKED | LV_STATE_PRESSED); + lv_obj_add_event_cb(camera_switch_track_, camera_toggle_event, LV_EVENT_CLICKED, this); + camera_switch_knob_ = make_shape(camera_switch_track_, 0, 0, 32, 32, 16); + lv_obj_set_style_bg_opa(camera_switch_knob_, LV_OPA_COVER, 0); + lv_obj_set_style_bg_color(camera_switch_knob_, lv_color_hex(COLOR_MUTED), 0); + lv_obj_align(camera_switch_knob_, LV_ALIGN_LEFT_MID, 5, 0); gesture_progress_ = lv_bar_create(screen); lv_obj_remove_style_all(gesture_progress_); lv_obj_set_pos(gesture_progress_, MARGIN, HEADER_H - 4); @@ -843,7 +906,8 @@ void DisplayUi::build_settings(lv_obj_t* screen) lv_obj_add_flag(settings_panel_, LV_OBJ_FLAG_HIDDEN); } -bool DisplayUi::apply_settings(const PanelSettings& settings, bool visible, bool save_error, bool modified) +bool DisplayUi::apply_settings(const DisplaySettings& settings, bool camera, bool visible, + bool save_error, bool modified) { DisplayLock lock; if (!lock) return false; @@ -854,8 +918,6 @@ bool DisplayUi::apply_settings(const PanelSettings& settings, bool visible, bool return false; brightness_percent_ = settings.brightness; } - if (gesture_toggle_label_) - lv_label_set_text(gesture_toggle_label_, settings.gestures ? "HAND GESTURES / ON" : "HAND GESTURES / OFF"); const bool was_visible = settings_visible_.exchange(visible, std::memory_order_relaxed); lv_label_set_text_fmt(settings_brightness_, "%u%%", settings.brightness); lv_obj_set_state(settings_buttons_[0], LV_STATE_DISABLED, settings.brightness <= 1); @@ -871,6 +933,11 @@ bool DisplayUi::apply_settings(const PanelSettings& settings, bool visible, bool else lv_label_set_text_fmt(settings_timeout_, "AFTER IDLE / MIN - CURRENT %lu SEC", static_cast(settings.clock_timeout_seconds)); lv_label_set_text(settings_hint_, save_error ? "SAVE FAILED - TRY AGAIN" : ""); + if (camera_switch_track_) { + lv_obj_set_state(camera_switch_track_, LV_STATE_CHECKED, camera); + lv_obj_set_style_bg_color(camera_switch_knob_, lv_color_hex(camera ? COLOR_BG : COLOR_MUTED), 0); + lv_obj_align(camera_switch_knob_, camera ? LV_ALIGN_RIGHT_MID : LV_ALIGN_LEFT_MID, camera ? -5 : 5, 0); + } if (visible && !was_visible) { lv_obj_remove_flag(settings_panel_, LV_OBJ_FLAG_HIDDEN); lv_obj_move_foreground(settings_panel_); @@ -878,33 +945,112 @@ bool DisplayUi::apply_settings(const PanelSettings& settings, bool visible, bool return true; } -void DisplayUi::gesture_toggle_event(lv_event_t* event) +void DisplayUi::camera_toggle_event(lv_event_t* event) +{ + auto* ui = static_cast(lv_event_get_user_data(event)); + if (ui->settings_visible_.load(std::memory_order_relaxed)) + ui->camera_toggle_.store(true, std::memory_order_relaxed); +} + +void DisplayUi::camera_preview_toggle_event(lv_event_t* event) { auto* ui = static_cast(lv_event_get_user_data(event)); - if (ui->settings_visible_.load()) ui->gesture_toggle_ = true; + if (!ui->camera_preview_ || !ui->camera_preview_available_) return; + const bool enabled = !ui->camera_preview_enabled_.load(std::memory_order_relaxed); + ui->camera_preview_enabled_.store(enabled, std::memory_order_relaxed); + lv_obj_set_flag(ui->camera_preview_, LV_OBJ_FLAG_HIDDEN, !enabled); } -void DisplayUi::show_gesture_status(GestureStatus status, uint8_t progress, bool visible) +bool DisplayUi::show_camera_state(bool enabled, GestureStatus status, uint8_t progress, bool visible) { - if (!gesture_status_label_) return; + if (!camera_indicator_) return true; DisplayLock lock; - if (!lock) return; - const char* text = ""; - if (visible) { - if (status == GestureStatus::Starting) text = "CAMERA STARTING"; - else if (status == GestureStatus::CameraError) text = "CHECK LEFT CAMERA / RETRY IN SETTINGS"; - else if (status == GestureStatus::MemoryError) text = "GESTURES UNAVAILABLE / LOW MEMORY"; - else if (status == GestureStatus::Fault) text = "GESTURES UNAVAILABLE / RESTART DEVICE"; - else if (progress) text = "HOLD OK TO ENTER CLOCK"; - } - lv_label_set_text(gesture_status_label_, text); - lv_obj_set_flag(gesture_status_label_, LV_OBJ_FLAG_HIDDEN, !text[0]); + if (!lock) return false; + // Text-bright while the sensor is starting or capturing; dim while the switch + // is on but capture is paused (settings, pairing, OTA); dim and struck + // through when capture failed and needs a switch-off/on retry or a reboot. + const bool capturing = status == GestureStatus::Starting || status == GestureStatus::Ready; + const bool failed = status == GestureStatus::CameraError || status == GestureStatus::MemoryError || + status == GestureStatus::Fault; + const lv_color_t color = lv_color_hex(capturing ? COLOR_TEXT : COLOR_DIM); + lv_obj_set_flag(camera_indicator_, LV_OBJ_FLAG_HIDDEN, !enabled); + for (lv_obj_t* part : camera_indicator_parts_) lv_obj_set_style_bg_color(part, color, 0); + for (lv_obj_t* slash : camera_indicator_slash_) lv_obj_set_flag(slash, LV_OBJ_FLAG_HIDDEN, !failed); lv_obj_set_flag(gesture_progress_, LV_OBJ_FLAG_HIDDEN, !visible || !progress); - if (text[0]) lv_obj_move_foreground(gesture_status_label_); + camera_preview_available_ = enabled && visible && status == GestureStatus::Ready; + if (camera_preview_) + lv_obj_set_flag(camera_preview_, LV_OBJ_FLAG_HIDDEN, + !camera_preview_available_ || !camera_preview_enabled_.load(std::memory_order_relaxed)); if (visible && progress) { lv_bar_set_value(gesture_progress_, progress, LV_ANIM_OFF); lv_obj_move_foreground(gesture_progress_); } + return true; +} + +void DisplayUi::build_camera_preview(lv_obj_t* screen) +{ +#if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW + // Half-resolution RGB preview; either rotated dimension is at most 160 pixels. + constexpr uint32_t kSide = 160; + const uint32_t stride = lv_draw_buf_width_to_stride(kSide, LV_COLOR_FORMAT_RGB565); + camera_preview_pixels_ = static_cast( + heap_caps_aligned_calloc(LV_DRAW_BUF_ALIGN, stride, kSide, MALLOC_CAP_SPIRAM)); + if (!camera_preview_pixels_) return; + camera_preview_ = make_shape(screen, SCREEN_W - MARGIN - kSide, CONTENT_Y + 8, kSide, kSide, 0); + camera_preview_canvas_ = lv_canvas_create(camera_preview_); + lv_canvas_set_buffer(camera_preview_canvas_, camera_preview_pixels_, kSide, 120, LV_COLOR_FORMAT_RGB565); + lv_obj_set_pos(camera_preview_canvas_, 0, 0); + camera_preview_box_ = make_shape(camera_preview_, 0, 0, 1, 1, 0); + lv_obj_set_style_border_width(camera_preview_box_, 2, 0); + lv_obj_set_style_border_color(camera_preview_box_, lv_color_hex(COLOR_TEXT), 0); + lv_obj_add_flag(camera_preview_box_, LV_OBJ_FLAG_HIDDEN); + camera_preview_label_ = make_label(camera_preview_, "", 2, 2, &montserrat_semibold_13, COLOR_TEXT); + lv_obj_set_style_bg_color(camera_preview_label_, lv_color_hex(COLOR_BG), 0); + lv_obj_set_style_bg_opa(camera_preview_label_, LV_OPA_70, 0); + lv_obj_add_flag(camera_preview_, LV_OBJ_FLAG_HIDDEN); +#else + (void)screen; +#endif +} + +void DisplayUi::show_camera_preview(const uint8_t* rgb, unsigned width, unsigned height, const HandObservation& hand) +{ + if (!camera_preview_enabled_.load(std::memory_order_relaxed) || + !camera_preview_canvas_ || !rgb || width > 320 || height > 320) return; + const uint32_t w = width / 2; + const uint32_t h = height / 2; + DisplayLock lock; + if (!lock) return; + if (lv_obj_has_flag(camera_preview_, LV_OBJ_FLAG_HIDDEN)) return; + const lv_image_dsc_t* image = lv_canvas_get_image(camera_preview_canvas_); + if (image->header.w != w || image->header.h != h) { + lv_canvas_set_buffer(camera_preview_canvas_, camera_preview_pixels_, w, h, LV_COLOR_FORMAT_RGB565); + lv_obj_set_size(camera_preview_, w, h); + lv_obj_set_x(camera_preview_, SCREEN_W - MARGIN - w); + } + const uint32_t stride = lv_canvas_get_image(camera_preview_canvas_)->header.stride; + for (uint32_t y = 0; y < h; ++y) { + auto* row = reinterpret_cast(camera_preview_pixels_ + y * stride); + for (uint32_t x = 0; x < w; ++x) { + const uint8_t* p = rgb + (size_t(y) * 2 * width + x * 2) * 3; + row[x] = uint16_t(((p[0] & 0xF8) << 8) | ((p[1] & 0xFC) << 3) | (p[2] >> 3)); + } + } + // The image cache is disabled (CONFIG_LV_CACHE_DEF_SIZE=0), so invalidation redraws the new pixels. + lv_obj_invalidate(camera_preview_canvas_); + if (hand.count) { + const int bw = int(hand.width * w); + const int bh = int(hand.height * h); + lv_obj_set_pos(camera_preview_box_, int(hand.x * w) - bw / 2, int(hand.y * h) - bh / 2); + lv_obj_set_size(camera_preview_box_, bw, bh); + lv_obj_remove_flag(camera_preview_box_, LV_OBJ_FLAG_HIDDEN); + lv_label_set_text_fmt(camera_preview_label_, "N%u S%d OK%d", hand.count, int(hand.score * 100), + int(hand.ok_score * 100)); + } else { + lv_obj_add_flag(camera_preview_box_, LV_OBJ_FLAG_HIDDEN); + lv_label_set_text(camera_preview_label_, "NO HAND"); + } } void DisplayUi::settings_event(lv_event_t* event) diff --git a/firmware/targets/esp_mosaico/main/display_ui.hpp b/firmware/targets/esp_mosaico/main/display_ui.hpp index 06ae0f8..cc8d82d 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.hpp +++ b/firmware/targets/esp_mosaico/main/display_ui.hpp @@ -6,7 +6,6 @@ #include "esp_lcd_panel_io.h" #include "lvgl.h" #include "orientation.hpp" -#include "panel_settings.hpp" #include "gesture_input.hpp" #include "usage_panel_state/display_settings.hpp" #include "usage_panel_state/page_state.hpp" @@ -31,9 +30,21 @@ class DisplayUi { bool begin(ScreenRotation initial_rotation); void request_page(Page page) { requested_page_.store(page, std::memory_order_relaxed); } - bool apply_settings(const PanelSettings& settings, bool visible, bool save_error = false, bool modified = false); - bool take_gesture_toggle() { return gesture_toggle_.exchange(false); } - void show_gesture_status(GestureStatus status, uint8_t progress, bool visible); + /** Shows the settings preview; camera is the previewed switch state, committed by SAVE. */ + bool apply_settings(const DisplaySettings& settings, bool camera, bool visible, + bool save_error = false, bool modified = false); + /** Consumes one press of the settings camera switch. The switch never changes itself. */ + bool take_camera_toggle() { return camera_toggle_.exchange(false, std::memory_order_relaxed); } + /** + * Updates the header camera icon and the OK-hold progress bar. + * Returns false if the LVGL lock was unavailable, so the caller can retry. + */ + bool show_camera_state(bool enabled, GestureStatus status, uint8_t progress, bool feedback_visible); + /** + * Draws the model input at half size with the detected box (debug preview). + * Called from the vision worker; drawing requires an enabled, available preview. + */ + void show_camera_preview(const uint8_t* rgb, unsigned width, unsigned height, const HandObservation& hand); SettingsAction take_settings_action() { return settings_action_.exchange(SettingsAction::None, std::memory_order_relaxed); @@ -109,8 +120,10 @@ class DisplayUi { }; void build_settings(lv_obj_t* screen); + void build_camera_preview(lv_obj_t* screen); static void settings_event(lv_event_t* event); - static void gesture_toggle_event(lv_event_t* event); + static void camera_toggle_event(lv_event_t* event); + static void camera_preview_toggle_event(lv_event_t* event); static void long_press_event(lv_event_t* event); void build_header(lv_obj_t* screen); void build_overview(lv_obj_t* screen); @@ -145,11 +158,21 @@ class DisplayUi { lv_obj_t* settings_buttons_[9]{}; lv_obj_t* settings_timeout_ = nullptr; lv_obj_t* settings_hint_ = nullptr; - lv_obj_t* gesture_toggle_button_ = nullptr; - lv_obj_t* gesture_toggle_label_ = nullptr; - lv_obj_t* gesture_status_label_ = nullptr; + lv_obj_t* camera_switch_track_ = nullptr; + lv_obj_t* camera_switch_knob_ = nullptr; + lv_obj_t* camera_indicator_ = nullptr; + lv_obj_t* camera_indicator_parts_[3]{}; // Viewfinder bump, body and lens centre. + lv_obj_t* camera_indicator_lens_ = nullptr; // Background-coloured lens ring. + lv_obj_t* camera_indicator_slash_[2]{}; // Background halo and stroke. lv_obj_t* gesture_progress_ = nullptr; - std::atomic gesture_toggle_{false}; + std::atomic camera_preview_enabled_{false}; + bool camera_preview_available_ = false; // Protected by the display lock. + lv_obj_t* camera_preview_ = nullptr; + lv_obj_t* camera_preview_canvas_ = nullptr; + lv_obj_t* camera_preview_box_ = nullptr; + lv_obj_t* camera_preview_label_ = nullptr; + uint8_t* camera_preview_pixels_ = nullptr; // RGB565 canvas buffer in PSRAM. + std::atomic camera_toggle_{false}; uint8_t brightness_percent_ = 50; // Serialized by the LVGL lock. std::atomic requested_page_{Page::Overview}; std::atomic ota_active_{false}; diff --git a/firmware/targets/esp_mosaico/main/gesture_camera.cpp b/firmware/targets/esp_mosaico/main/gesture_camera.cpp index 1319342..31e3881 100644 --- a/firmware/targets/esp_mosaico/main/gesture_camera.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_camera.cpp @@ -14,6 +14,8 @@ #include "esp_video_device.h" #include "esp_video_init.h" #include "esp_video_ioctl.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" #include "hal/usb_serial_jtag_ll.h" #include "linux/videodev2.h" @@ -22,6 +24,17 @@ namespace { constexpr char TAG[] = "gesture_camera"; constexpr gpio_num_t kPowerDown = GPIO_NUM_48; constexpr gpio_num_t kFlash = GPIO_NUM_34; +// Start-up timing follows esp-vision boards/ESP32_S31_MOSAICO/camera.c. +constexpr uint32_t kPowerStabilizationMs = 20; +constexpr suseconds_t kFrameTimeoutUs = 500000; +constexpr unsigned kPrimeAttempts = 3; +// esp-mosaico-bsp mosaico_module_camera and its module manager access the +// subboard I2C1 bus, which has only internal pull-ups, at 100 kHz. +constexpr uint32_t kSccbFrequencyHz = 100000; +constexpr uint16_t kModuleEepromAddress = 0x50; +constexpr uint16_t kOv3640Address = 0x3c; +constexpr uint16_t kSc101iotAddress = 0x68; +constexpr int kProbeTimeoutMs = 50; bool checked_ioctl(int fd, unsigned long command, void* arg) { @@ -31,10 +44,19 @@ bool checked_ioctl(int fd, unsigned long command, void* arg) } } +bool GestureCamera::fail(const char* step, int code) +{ + failure_ = step; + failure_code_ = code; + return false; +} + bool GestureCamera::start() { - if (pins_claimed_ || fd_ >= 0) return false; - if (bsp_subboard_init() != ESP_OK) return false; + failure_ = nullptr; + failure_code_ = 0; + if (pins_claimed_ || fd_ >= 0) return fail("start while resources are held"); + if (const esp_err_t error = bsp_subboard_init(); error != ESP_OK) return fail("subboard init", error); // D2 shares GPIO33 with Serial/JTAG. The UART console remains available. restore_usb_pad_ = usb_serial_jtag_ll_phy_is_pad_enabled(); restore_usb_clock_ = usb_serial_jtag_ll_module_is_enabled(); @@ -48,11 +70,23 @@ bool GestureCamera::start() // The illuminator is active-low and must never be driven by gesture capture. gpio_set_level(kFlash, 1); gpio_set_direction(kFlash, GPIO_MODE_OUTPUT); + // The CameraBoard oscillator and sensor need time after the slot is claimed + // before the first SCCB access in esp_video_init. + vTaskDelay(pdMS_TO_TICKS(kPowerStabilizationMs)); + // Diagnostic only: the sensor may still be held in power-down before + // esp_video_init drives PWDN, so a missing sensor ACK here is not an error. + i2c_master_bus_handle_t sccb = bsp_subboard_get_i2c_bus(); + const auto ack = [&](uint16_t address) { + return i2c_master_probe(sccb, address, kProbeTimeoutMs) == ESP_OK ? "ack" : "no ack"; + }; + const bool eeprom_ack = i2c_master_probe(sccb, kModuleEepromAddress, kProbeTimeoutMs) == ESP_OK; + ESP_LOGI(TAG, "subboard I2C probe: module EEPROM 0x50 %s, OV3640 0x3c %s, SC101IOT 0x68 %s", + eeprom_ack ? "ack" : "no ack", ack(kOv3640Address), ack(kSc101iotAddress)); esp_video_init_dvp_config_t dvp{}; dvp.sccb_config.init_sccb = false; - dvp.sccb_config.i2c_handle = bsp_subboard_get_i2c_bus(); - dvp.sccb_config.freq = 400000; + dvp.sccb_config.i2c_handle = sccb; + dvp.sccb_config.freq = kSccbFrequencyHz; dvp.reset_pin = GPIO_NUM_53; dvp.pwdn_pin = kPowerDown; dvp.dvp_pin.data_width = CAM_CTLR_DATA_WIDTH_8; @@ -68,47 +102,70 @@ bool GestureCamera::start() config.dvp = &dvp; const esp_err_t error = esp_video_init_with_flags(&config, ESP_VIDEO_INIT_FLAGS_DVP); if (error != ESP_OK) { - ESP_LOGW(TAG, "left-slot OV3640 unavailable: %s", esp_err_to_name(error)); - return false; + ESP_LOGW(TAG, "left-slot camera unavailable: %s", esp_err_to_name(error)); + return fail("esp_video_init (sensor not detected?)", error); } video_initialized_ = true; fd_ = open(ESP_VIDEO_DVP_DEVICE_NAME, O_RDWR); - if (fd_ < 0) return false; + // esp_video_init succeeds without registering the device when sensor + // detection fails; code reports whether the module EEPROM answered. + if (fd_ < 0 && errno == ENOENT) + return fail("no sensor detected (OV3640 0x3c / SC101IOT 0x68)", eeprom_ack ? 1 : 0); + if (fd_ < 0) return fail("open video device", errno); v4l2_format format{}; format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - format.fmt.pix.width = 640; - format.fmt.pix.height = 480; + // Preserve the detected sensor's configured field of view, as in mosaico_module_camera. + if (!checked_ioctl(fd_, VIDIOC_G_FMT, &format)) return fail("get format", errno); + const unsigned requested_width = format.fmt.pix.width; + const unsigned requested_height = format.fmt.pix.height; + if (!((requested_width == 1280 && requested_height == 720) || + (requested_width == 640 && requested_height == 480))) return fail("unsupported frame size"); format.fmt.pix.pixelformat = V4L2_PIX_FMT_UYVY; - if (!checked_ioctl(fd_, VIDIOC_S_FMT, &format)) return false; + if (!checked_ioctl(fd_, VIDIOC_S_FMT, &format)) return fail("set format", errno); if (format.fmt.pix.pixelformat != V4L2_PIX_FMT_UYVY || - format.fmt.pix.width != 640 || format.fmt.pix.height != 480) return false; + format.fmt.pix.width != requested_width || format.fmt.pix.height != requested_height) return fail("format mismatch"); width_ = format.fmt.pix.width; height_ = format.fmt.pix.height; stride_ = format.fmt.pix.bytesperline; if (!stride_) stride_ = width_ * 2; - timeval timeout{0, 200000}; - if (!checked_ioctl(fd_, VIDIOC_S_DQBUF_TIMEOUT, &timeout)) return false; + timeval timeout{0, kFrameTimeoutUs}; + if (!checked_ioctl(fd_, VIDIOC_S_DQBUF_TIMEOUT, &timeout)) return fail("set dequeue timeout", errno); v4l2_requestbuffers request{}; request.count = 2; request.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; request.memory = V4L2_MEMORY_MMAP; - if (!checked_ioctl(fd_, VIDIOC_REQBUFS, &request) || request.count != 2) return false; + if (!checked_ioctl(fd_, VIDIOC_REQBUFS, &request) || request.count != 2) + return fail("request buffers", errno); for (unsigned i = 0; i < 2; ++i) { v4l2_buffer buffer{}; buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; buffer.memory = V4L2_MEMORY_MMAP; buffer.index = i; - if (!checked_ioctl(fd_, VIDIOC_QUERYBUF, &buffer)) return false; + if (!checked_ioctl(fd_, VIDIOC_QUERYBUF, &buffer)) return fail("query buffer", errno); buffers_[i] = mmap(nullptr, buffer.length, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, buffer.m.offset); - if (buffers_[i] == MAP_FAILED) { buffers_[i] = nullptr; return false; } + if (buffers_[i] == MAP_FAILED) { buffers_[i] = nullptr; return fail("map buffer", errno); } lengths_[i] = buffer.length; - if (!checked_ioctl(fd_, VIDIOC_QBUF, &buffer)) return false; + if (!checked_ioctl(fd_, VIDIOC_QBUF, &buffer)) return fail("queue buffer", errno); } int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; streaming_ = checked_ioctl(fd_, VIDIOC_STREAMON, &type); - if (streaming_) ESP_LOGI(TAG, "OV3640 UYVY %ux%u stride=%u buffers=2", width_, height_, stride_); - return streaming_; + if (!streaming_) return fail("stream on", errno); + // The first frames arrive only after sensor start-up. Discard one per + // buffer, allowing a few frame timeouts before reporting a dead stream. + for (unsigned primed = 0, attempts = 0; primed < 2;) { + v4l2_buffer buffer{}; + buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buffer.memory = V4L2_MEMORY_MMAP; + if (ioctl(fd_, VIDIOC_DQBUF, &buffer) != 0) { + if (++attempts >= kPrimeAttempts) return fail("prime stream (no frames)", errno); + continue; + } + if (!checked_ioctl(fd_, VIDIOC_QBUF, &buffer)) return fail("prime stream requeue", errno); + ++primed; + } + ESP_LOGI(TAG, "camera UYVY %ux%u stride=%u buffers=2", width_, height_, stride_); + return true; } bool GestureCamera::read_rgb(uint8_t* output, unsigned rotation, bool mirror) @@ -116,7 +173,8 @@ bool GestureCamera::read_rgb(uint8_t* output, unsigned rotation, bool mirror) v4l2_buffer buffer{}; buffer.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; buffer.memory = V4L2_MEMORY_MMAP; - if (!streaming_ || !checked_ioctl(fd_, VIDIOC_DQBUF, &buffer)) return false; + if (!streaming_) return fail("read while stopped"); + if (!checked_ioctl(fd_, VIDIOC_DQBUF, &buffer)) return fail("dequeue frame (no frames?)", errno); // At most two capture buffers exist. Prefer the newer completed frame if // inference left both queued; never accumulate camera latency across cycles. timeval nonblocking{}; @@ -130,39 +188,42 @@ bool GestureCamera::read_rgb(uint8_t* output, unsigned rotation, bool mirror) buffer = newer; timeout_ok = returned; } - timeval blocking{0, 200000}; + timeval blocking{0, kFrameTimeoutUs}; timeout_ok = checked_ioctl(fd_, VIDIOC_S_DQBUF_TIMEOUT, &blocking) && timeout_ok; } bool converted = false; - if (buffer.index < 2 && buffers_[buffer.index]) { + if (buffer.index < 2 && buffers_[buffer.index] && !(buffer.flags & V4L2_BUF_FLAG_ERROR)) { const size_t bytes = buffer.bytesused ? buffer.bytesused : lengths_[buffer.index]; if (bytes <= lengths_[buffer.index]) converted = prepare_gesture_image(static_cast(buffers_[buffer.index]), bytes, width_, height_, stride_, output, rotation, mirror); } const bool returned = checked_ioctl(fd_, VIDIOC_QBUF, &buffer); - return converted && returned && timeout_ok; + if (!converted) return fail("convert frame", int(buffer.bytesused)); + if (!returned || !timeout_ok) return fail("requeue frame", errno); + return true; } bool GestureCamera::stop() { if (streaming_) { int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (!checked_ioctl(fd_, VIDIOC_STREAMOFF, &type)) return false; + if (!checked_ioctl(fd_, VIDIOC_STREAMOFF, &type)) return fail("stream off", errno); streaming_ = false; } if (fd_ >= 0) { for (unsigned i = 0; i < 2; ++i) { if (buffers_[i]) { - if (munmap(buffers_[i], lengths_[i]) != 0) return false; + if (munmap(buffers_[i], lengths_[i]) != 0) return fail("unmap buffer", errno); buffers_[i] = nullptr; } } - if (close(fd_) != 0) return false; + if (close(fd_) != 0) return fail("close video device", errno); fd_ = -1; } if (video_initialized_) { - if (esp_video_deinit_with_flags(ESP_VIDEO_INIT_FLAGS_DVP) != ESP_OK) return false; + if (const esp_err_t error = esp_video_deinit_with_flags(ESP_VIDEO_INIT_FLAGS_DVP); error != ESP_OK) + return fail("esp_video_deinit", error); video_initialized_ = false; } if (pins_claimed_) { @@ -175,7 +236,8 @@ bool GestureCamera::stop() usb_serial_jtag_ll_phy_enable_pad(restore_usb_pad_); if (restore_usb_clock_ && restore_usb_interrupts_) usb_serial_jtag_ll_ena_intr_mask(restore_usb_interrupts_); - if (bsp_subboard_apply_address_select(BSP_SUBBOARD_SLOT_LEFT) != ESP_OK) return false; + if (const esp_err_t error = bsp_subboard_apply_address_select(BSP_SUBBOARD_SLOT_LEFT); error != ESP_OK) + return fail("restore address select", error); pins_claimed_ = false; } return true; diff --git a/firmware/targets/esp_mosaico/main/gesture_camera.hpp b/firmware/targets/esp_mosaico/main/gesture_camera.hpp index 21b98f8..a16ea83 100644 --- a/firmware/targets/esp_mosaico/main/gesture_camera.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_camera.hpp @@ -1,20 +1,31 @@ #pragma once +#include "gesture_image.hpp" + #include #include namespace usage_panel::mosaico { -/** Owns the left-slot OV3640 and its V4L2 buffers on the vision worker only. */ +/** Owns the left-slot CameraBoard (OV3640 or SC101IOT) and its V4L2 buffers on the vision worker only. */ class GestureCamera { public: bool start(); /** Copies/converts one frame and requeues it before returning to the caller. */ bool read_rgb(uint8_t* output, unsigned rotation, bool mirror); + /** Packed RGB dimensions for the active capture format and rotation. */ + GestureImageSize image_size(unsigned rotation) const { return gesture_image_size(width_, height_, rotation); } /** Returns false if resources cannot be safely released; do not restart afterward. */ bool stop(); + /** Names the step of the most recent failed call, or nullptr; code is errno or esp_err_t. */ + const char* failure() const { return failure_; } + int failure_code() const { return failure_code_; } private: + bool fail(const char* step, int code = 0); + + const char* failure_ = nullptr; + int failure_code_ = 0; int fd_ = -1; bool pins_claimed_ = false; bool video_initialized_ = false; diff --git a/firmware/targets/esp_mosaico/main/gesture_image.cpp b/firmware/targets/esp_mosaico/main/gesture_image.cpp index 67996d4..45bbc3a 100644 --- a/firmware/targets/esp_mosaico/main/gesture_image.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_image.cpp @@ -10,6 +10,19 @@ uint8_t channel(int value) } } +GestureImageSize gesture_image_size(unsigned width, unsigned height, unsigned turns) +{ + if (width < 2 || width > 4096 || (width & 1) || height == 0 || height > 4096 || turns > 3) + return {}; + GestureImageSize size{kGestureImageWidth, kGestureImageHeight}; + if (width * kGestureImageHeight >= height * kGestureImageWidth) + size.height = std::max(1U, height * kGestureImageWidth / width); + else + size.width = std::max(1U, width * kGestureImageHeight / height); + if (turns & 1) std::swap(size.width, size.height); + return size; +} + bool prepare_gesture_image(const uint8_t* src, size_t size, unsigned width, unsigned height, unsigned stride, uint8_t* dst, unsigned turns, bool mirror) @@ -17,22 +30,23 @@ bool prepare_gesture_image(const uint8_t* src, size_t size, unsigned width, if (!src || !dst || width < 2 || width > 4096 || height == 0 || height > 4096 || (width & 1) || stride < width * 2 || size < size_t(stride) * height || turns > 3) return false; - const unsigned out_width = turns & 1 ? kGestureImageHeight : kGestureImageWidth; - for (unsigned y = 0; y < kGestureImageHeight; ++y) { - for (unsigned x = 0; x < kGestureImageWidth; ++x) { - const unsigned sx = x * width / kGestureImageWidth; - const unsigned sy = y * height / kGestureImageHeight; + const auto size_out = gesture_image_size(width, height, 0); + const unsigned out_width = turns & 1 ? size_out.height : size_out.width; + for (unsigned y = 0; y < size_out.height; ++y) { + for (unsigned x = 0; x < size_out.width; ++x) { + const unsigned sx = x * width / size_out.width; + const unsigned sy = y * height / size_out.height; const uint8_t* p = src + size_t(sy) * stride + (sx & ~1U) * 2; const int luma = std::max(0, int(p[(sx & 1) ? 3 : 1]) - 16); const int u = int(p[0]) - 128; const int v = int(p[2]) - 128; - const unsigned mx = mirror ? kGestureImageWidth - 1 - x : x; + const unsigned mx = mirror ? size_out.width - 1 - x : x; unsigned ox = mx, oy = y; - if (turns == 1) { ox = kGestureImageHeight - 1 - y; oy = mx; } - if (turns == 2) { ox = kGestureImageWidth - 1 - mx; oy = kGestureImageHeight - 1 - y; } - if (turns == 3) { ox = y; oy = kGestureImageWidth - 1 - mx; } + if (turns == 1) { ox = size_out.height - 1 - y; oy = mx; } + if (turns == 2) { ox = size_out.width - 1 - mx; oy = size_out.height - 1 - y; } + if (turns == 3) { ox = y; oy = size_out.width - 1 - mx; } uint8_t* rgb = dst + (oy * out_width + ox) * 3; - // OV3640 UYVY uses the limited-range BT.601 YCbCr conversion. + // UYVY is converted with limited-range BT.601 YCbCr coefficients. rgb[0] = channel(298 * luma + 409 * v + 128); rgb[1] = channel(298 * luma - 100 * u - 208 * v + 128); rgb[2] = channel(298 * luma + 516 * u + 128); diff --git a/firmware/targets/esp_mosaico/main/gesture_image.hpp b/firmware/targets/esp_mosaico/main/gesture_image.hpp index f403f17..cb1a961 100644 --- a/firmware/targets/esp_mosaico/main/gesture_image.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_image.hpp @@ -9,8 +9,19 @@ constexpr unsigned kGestureImageWidth = 320; constexpr unsigned kGestureImageHeight = 240; constexpr size_t kGestureImageBytes = kGestureImageWidth * kGestureImageHeight * 3; +struct GestureImageSize { + unsigned width; + unsigned height; +}; + +/** Fits the complete frame within 320x240, preserving aspect ratio before rotation. + * Returns zero dimensions for invalid input or an unsupported rotation. + */ +GestureImageSize gesture_image_size(unsigned width, unsigned height, unsigned quarter_turns); + /** Downsamples packed UYVY to RGB888, mirrors horizontally, then rotates clockwise. - * Output capacity must be at least kGestureImageBytes. Odd rotations yield 240x320. + * Output capacity must be at least kGestureImageBytes. Actual packed dimensions + * are returned by gesture_image_size(); no cropping or padding is applied. */ bool prepare_gesture_image(const uint8_t* src, size_t size, unsigned width, unsigned height, unsigned stride, uint8_t* dst, diff --git a/firmware/targets/esp_mosaico/main/gesture_input.cpp b/firmware/targets/esp_mosaico/main/gesture_input.cpp index d25ea10..bb98402 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.cpp @@ -27,6 +27,9 @@ constexpr uint32_t kAsleep = 4; constexpr uint32_t kContextMask = 0x7f; constexpr uint32_t kGeneration = 0x80; constexpr char TAG[] = "gesture_input"; +// Transient start-up failures are retried as in esp-vision's camera init. +constexpr unsigned kCameraStartAttempts = 3; +constexpr uint32_t kCameraRetryDelayMs = 100; } bool GestureInput::begin() @@ -89,9 +92,21 @@ void GestureInput::run() bool fatal_cleanup = false; unsigned capture_failures = 0; uint64_t next_report = 0; + uint64_t next_detection_log = 0; unsigned measured_frames = 0; uint64_t measured_us = 0; + uint64_t capture_us = 0, detect_us = 0, classify_us = 0, feedback_us = 0; unsigned warmup = 0; + // Start-up failures scroll past during boot, so the reason is repeated + // after cleanup until the switch is turned off. + const char* failure = nullptr; + int failure_code = 0; + uint64_t next_failure_log = 0; + const auto record = [&](const char* step, int code) { + failure = step ? step : "unknown"; + failure_code = code; + next_failure_log = 0; + }; for (;;) { const uint32_t context = context_.load(); @@ -106,6 +121,7 @@ void GestureInput::run() if (active || rgb) { if (!camera.stop()) { ESP_LOGE(TAG, "camera cleanup failed; recognition disabled until reboot"); + record(camera.failure(), camera.failure_code()); fatal_cleanup = true; fault = true; } @@ -116,7 +132,12 @@ void GestureInput::run() active = false; } progress_ = 0; - if (!enabled && !fatal_cleanup) { fault = false; status_ = GestureStatus::Off; } + const uint64_t now_ms = esp_timer_get_time() / 1000; + if (fault && failure && now_ms >= next_failure_log) { + ESP_LOGW(TAG, "capture unavailable: %s (code %d)", failure, failure_code); + next_failure_log = now_ms + 5000; + } + if (!enabled && !fatal_cleanup) { fault = false; failure = nullptr; status_ = GestureStatus::Off; } else if (fatal_cleanup) status_ = GestureStatus::Fault; else if (!fault) status_ = GestureStatus::Paused; vTaskDelay(pdMS_TO_TICKS(100)); @@ -125,23 +146,49 @@ void GestureInput::run() if (!active) { status_ = GestureStatus::Starting; // ESP-DL allocates model arenas during construction. Keep a conservative - // margin for both arenas, camera buffers and the concurrently running UI. - if (heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM) < 6 * 1024 * 1024 || + // margin for both arenas, two 1280x720 UYVY buffers and the concurrently running UI. + if (heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM) < 9 * 1024 * 1024 || heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT) < 96 * 1024) { + // Code is the largest free PSRAM block in KiB. + record("memory precheck", + int(heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM) / 1024)); status_ = GestureStatus::MemoryError; fault = true; continue; } rgb = static_cast(heap_caps_malloc(kGestureImageBytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); - if (!rgb) { status_ = GestureStatus::MemoryError; fault = true; continue; } - if (!camera.start()) { - status_ = GestureStatus::CameraError; + if (!rgb) { + record("frame buffer allocation", int(kGestureImageBytes)); + status_ = GestureStatus::MemoryError; + fault = true; + continue; + } + bool started = false; + for (unsigned attempt = 1; attempt <= kCameraStartAttempts; ++attempt) { + if (camera.start()) { started = true; break; } + record(camera.failure(), camera.failure_code()); + ESP_LOGW(TAG, "camera start %u/%u failed: %s (code %d)", attempt, + kCameraStartAttempts, failure, failure_code); + if (!camera.stop()) { + record(camera.failure(), camera.failure_code()); + fatal_cleanup = true; + break; + } + if (attempt < kCameraStartAttempts) vTaskDelay(pdMS_TO_TICKS(kCameraRetryDelayMs)); + } + if (!started) { + status_ = fatal_cleanup ? GestureStatus::Fault : GestureStatus::CameraError; fault = true; continue; } detector.reset(new (std::nothrow) HandDetect(HandDetect::ESPDET_PICO_224_224_HAND, false)); recognizer.reset(new (std::nothrow) HandGestureRecognizer()); - if (!detector || !recognizer) { status_ = GestureStatus::MemoryError; fault = true; continue; } + if (!detector || !recognizer) { + record("model allocation", 0); + status_ = GestureStatus::MemoryError; + fault = true; + continue; + } detector->set_score_thr(0.5F); tracker.reset(); active = true; @@ -163,18 +210,25 @@ void GestureInput::run() if (!camera.read_rgb(rgb, rotation, mirror)) { tracker.reset(); progress_ = 0; - if (++capture_failures >= 3) { fault = true; status_ = GestureStatus::CameraError; } + if (++capture_failures >= 3) { + record(camera.failure(), camera.failure_code()); + fault = true; + status_ = GestureStatus::CameraError; + } vTaskDelay(pdMS_TO_TICKS(20)); continue; } capture_failures = 0; if (warmup) { --warmup; continue; } if (context != context_.load()) continue; + const auto image_size = camera.image_size(rotation); dl::image::img_t image{rgb, - static_cast(rotation & 1 ? kGestureImageHeight : kGestureImageWidth), - static_cast(rotation & 1 ? kGestureImageWidth : kGestureImageHeight), + static_cast(image_size.width), + static_cast(image_size.height), dl::image::DL_IMAGE_PIX_TYPE_RGB888}; + const uint64_t captured_us = esp_timer_get_time(); const auto& hands = detector->run(image); + const uint64_t detected_us = esp_timer_get_time(); HandObservation observation{}; observation.count = hands.size(); if (hands.size() == 1) { @@ -200,10 +254,27 @@ void GestureInput::run() if (context != context_.load()) continue; const GestureAction action = tracker.update(observation, finished_us / 1000); progress_ = tracker.hold_progress(); +#if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW + if (preview_sink_) preview_sink_(preview_context_, rgb, image.width, image.height, observation); +#endif + // Calibration aid: normalized observations in percent, at most once per second. + if (observation.count && finished_us / 1000 >= next_detection_log) { + ESP_LOGI(TAG, "hand count=%u score=%d center=(%d,%d) size=(%d,%d) ok=%d", + observation.count, int(observation.score * 100), int(observation.x * 100), + int(observation.y * 100), int(observation.width * 100), + int(observation.height * 100), int(observation.ok_score * 100)); + next_detection_log = finished_us / 1000 + 1000; + } if (action != GestureAction::None) { + ESP_LOGI(TAG, "tracker action=%u", unsigned(action)); const GestureEvent event{action, context, finished_us / 1000}; xQueueSend(queue_, &event, 0); } + const uint64_t feedback_done_us = esp_timer_get_time(); + capture_us += captured_us - started_us; + detect_us += detected_us - captured_us; + classify_us += finished_us - detected_us; + feedback_us += feedback_done_us - finished_us; measured_us += finished_us - started_us; ++measured_frames; if (finished_us / 1000 >= next_report) { @@ -211,6 +282,12 @@ void GestureInput::run() measured_frames, static_cast(measured_us / measured_frames / 1000), unsigned(heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)), unsigned(heap_caps_get_free_size(MALLOC_CAP_SPIRAM)), unsigned(uxTaskGetStackHighWaterMark(nullptr))); + ESP_LOGI(TAG, "mean_stage_us capture_convert=%llu detect=%llu classify=%llu feedback=%llu", + static_cast(capture_us / measured_frames), + static_cast(detect_us / measured_frames), + static_cast(classify_us / measured_frames), + static_cast(feedback_us / measured_frames)); + capture_us = detect_us = classify_us = feedback_us = 0; measured_us = 0; measured_frames = 0; next_report = finished_us / 1000 + 10000; diff --git a/firmware/targets/esp_mosaico/main/gesture_input.hpp b/firmware/targets/esp_mosaico/main/gesture_input.hpp index 3a1c7b0..bab2b64 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.hpp @@ -20,6 +20,16 @@ struct GestureEvent { /** Worker-owned camera/model pipeline. The application owns context and event consumption. */ class GestureInput { public: + /** Receives each model input image and its observation on the vision worker. */ + using PreviewSink = void (*)(void* context, const uint8_t* rgb, unsigned width, unsigned height, + const HandObservation& hand); + + /** Must be called before begin(); only used with CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW. */ + void set_preview_sink(PreviewSink sink, void* context) + { + preview_sink_ = sink; + preview_context_ = context; + } bool begin(); void configure(bool enabled, bool allowed, bool asleep, Page page, ScreenRotation rotation); /** Cancels pending recognition after another input is accepted. Main task only. */ @@ -35,6 +45,8 @@ class GestureInput { std::atomic context_{0}; std::atomic status_{GestureStatus::Off}; std::atomic progress_{0}; + PreviewSink preview_sink_ = nullptr; + void* preview_context_ = nullptr; }; } // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp index 8133072..b0c0f1e 100644 --- a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp @@ -4,13 +4,19 @@ #include namespace usage_panel::mosaico { +namespace { +// Dropouts up to this long keep a swipe or OK hold alive. A longer gap restarts +// the track from the next usable sample; it does not require the hand to leave. +constexpr uint64_t kMaxSampleGapMs = 250; +} GestureEffect route_gesture(GestureAction action, bool allowed, bool asleep) { if (!allowed || action == GestureAction::None) return GestureEffect::None; if (action == GestureAction::EnterClock) return asleep ? GestureEffect::None : GestureEffect::Clock; if (asleep) return GestureEffect::Wake; - return action == GestureAction::SwipeLeft ? GestureEffect::Next : GestureEffect::Previous; + return action == GestureAction::SwipeRight || action == GestureAction::SwipeDown + ? GestureEffect::Next : GestureEffect::Previous; } void GestureTracker::reset() @@ -22,32 +28,40 @@ void GestureTracker::reset() GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) { - progress_ = 0; - if (h.count == 0) { - if (!absent_) { absent_ = true; absent_since_ = now; } - if (now - absent_since_ >= 300 && now - last_action_ >= 500) - waiting_release_ = false; - tracking_ = false; - holding_ok_ = false; + if (h.count > 1) { + // Several hands make the command owner ambiguous; wait for the view to clear. + absent_ = false; + tracking_ = holding_ok_ = false; + waiting_release_ = true; + progress_ = 0; return GestureAction::None; } - absent_ = false; - const bool valid = h.count == 1 && std::isfinite(h.x) && std::isfinite(h.y) && + const bool usable = h.count == 1 && std::isfinite(h.x) && std::isfinite(h.y) && std::isfinite(h.width) && std::isfinite(h.height) && std::isfinite(h.score) && h.score >= 0.65F && h.x >= 0 && h.x <= 1 && h.y >= 0 && h.y <= 1 && h.width >= 0.06F && h.height >= 0.06F && h.width <= 0.9F && h.height <= 0.95F; - if (!valid) { - tracking_ = false; - holding_ok_ = false; - waiting_release_ = true; + if (!usable) { + // Missed and low-confidence frames are dropouts, not new input. + if (h.count == 0) { + if (!absent_) { absent_ = true; absent_since_ = now; } + if (now - absent_since_ >= 300 && now - last_action_ >= 500) + waiting_release_ = false; + } else { + // A weakly detected hand is still present and cannot count as a release. + absent_ = false; + } + if (tracking_ && now - last_sample_ > kMaxSampleGapMs) tracking_ = holding_ok_ = false; + if (!holding_ok_) progress_ = 0; + return GestureAction::None; } + absent_ = false; + progress_ = 0; if (waiting_release_) return GestureAction::None; - if (tracking_ && (now < last_sample_ || now - last_sample_ > 250 || + if (tracking_ && (now < last_sample_ || now - last_sample_ > kMaxSampleGapMs || std::fabs(h.x - previous_.x) > std::max(0.18F, (h.width + previous_.width) * 0.75F) || std::fabs(h.y - previous_.y) > std::max(0.18F, (h.height + previous_.height) * 0.75F))) { - reset(); - return GestureAction::None; + tracking_ = holding_ok_ = false; } if (!tracking_) { tracking_ = true; @@ -55,21 +69,28 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) samples_ = 0; start_x_ = h.x; start_y_ = h.y; - path_x_ = max_dy_ = 0; + path_x_ = path_y_ = max_dx_ = max_dy_ = 0; previous_ = h; } ++samples_; path_x_ += std::fabs(h.x - previous_.x); + path_y_ += std::fabs(h.y - previous_.y); + max_dx_ = std::max(max_dx_, std::fabs(h.x - start_x_)); max_dy_ = std::max(max_dy_, std::fabs(h.y - start_y_)); const float dx = h.x - start_x_; + const float dy = h.y - start_y_; previous_ = h; last_sample_ = now; GestureAction action = GestureAction::None; - if (samples_ >= 3 && now - started_ >= 150 && now - started_ <= 900 && - std::fabs(dx) >= 0.25F && std::fabs(dx) >= 2 * max_dy_ && - max_dy_ <= 0.15F && std::fabs(dx) >= path_x_ * 0.75F) { - action = dx < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight; + if (samples_ >= 3 && now - started_ >= 150 && now - started_ <= 900) { + if (std::fabs(dx) >= 0.25F && std::fabs(dx) >= 2 * max_dy_ && + max_dy_ <= 0.15F && std::fabs(dx) >= path_x_ * 0.75F) { + action = dx < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight; + } else if (std::fabs(dy) >= 0.25F && std::fabs(dy) >= 2 * max_dx_ && + max_dx_ <= 0.15F && std::fabs(dy) >= path_y_ * 0.75F) { + action = dy < 0 ? GestureAction::SwipeUp : GestureAction::SwipeDown; + } } if (std::isfinite(h.ok_score) && h.ok_score >= 0.85F) { diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp index 55de43a..ddfb21c 100644 --- a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp @@ -4,7 +4,7 @@ namespace usage_panel::mosaico { -enum class GestureAction : uint8_t { None, SwipeLeft, SwipeRight, EnterClock }; +enum class GestureAction : uint8_t { None, SwipeLeft, SwipeRight, EnterClock, SwipeUp, SwipeDown }; enum class GestureEffect : uint8_t { None, Wake, Previous, Next, Clock }; /** Applies application input policy before any page or screensaver mutation. */ @@ -45,6 +45,8 @@ class GestureTracker { float start_x_ = 0; float start_y_ = 0; float path_x_ = 0; + float path_y_ = 0; + float max_dx_ = 0; float max_dy_ = 0; float ok_x_ = 0; float ok_y_ = 0; diff --git a/firmware/targets/esp_mosaico/main/panel_settings.cpp b/firmware/targets/esp_mosaico/main/panel_settings.cpp deleted file mode 100644 index 5c310ec..0000000 --- a/firmware/targets/esp_mosaico/main/panel_settings.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "panel_settings.hpp" -#include "nvs.h" - -namespace usage_panel::mosaico { - -PanelSettings load_panel_settings(DisplaySettings defaults) -{ - PanelSettings result{load_display_settings(defaults), false}; - nvs_handle_t handle; - if (nvs_open("display", NVS_READONLY, &handle) == ESP_OK) { - uint8_t enabled = 0; - if (nvs_get_u8(handle, "gestures", &enabled) == ESP_OK && enabled <= 1) - result.gestures = enabled != 0; - nvs_close(handle); - } - return result; -} - -esp_err_t save_panel_settings(const PanelSettings& settings) -{ - if (settings.brightness < 1 || settings.brightness > 100 || settings.clock_timeout_seconds > 86400) - return ESP_ERR_INVALID_ARG; - nvs_handle_t handle; - esp_err_t error = nvs_open("display", NVS_READWRITE, &handle); - if (error != ESP_OK) return error; - error = nvs_set_u8(handle, "brightness", settings.brightness); - if (error == ESP_OK) error = nvs_set_u32(handle, "clock_timeout", settings.clock_timeout_seconds); - if (error == ESP_OK) error = nvs_set_u8(handle, "gestures", settings.gestures); - if (error == ESP_OK) error = nvs_commit(handle); - nvs_close(handle); - return error; -} - -bool settings_differ(const PanelSettings& a, const PanelSettings& b) -{ - return a.brightness != b.brightness || a.clock_timeout_seconds != b.clock_timeout_seconds || - a.gestures != b.gestures; -} - -} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/panel_settings.hpp b/firmware/targets/esp_mosaico/main/panel_settings.hpp deleted file mode 100644 index 8fb91da..0000000 --- a/firmware/targets/esp_mosaico/main/panel_settings.hpp +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include "usage_panel_state/display_settings.hpp" - -namespace usage_panel::mosaico { - -struct PanelSettings : DisplaySettings { - bool gestures = false; -}; - -PanelSettings load_panel_settings(DisplaySettings defaults); -/** Stores display and local input preferences through a single NVS handle. */ -esp_err_t save_panel_settings(const PanelSettings& settings); -bool settings_differ(const PanelSettings& a, const PanelSettings& b); - -} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/sdkconfig.defaults b/firmware/targets/esp_mosaico/sdkconfig.defaults index 59a06bd..67a6219 100644 --- a/firmware/targets/esp_mosaico/sdkconfig.defaults +++ b/firmware/targets/esp_mosaico/sdkconfig.defaults @@ -18,6 +18,8 @@ CONFIG_BSP_CO5300_ENABLE_TE=y CONFIG_LV_COLOR_DEPTH_16=y CONFIG_LV_OS_FREERTOS=y CONFIG_LV_DEF_REFR_PERIOD=15 +# Clock glyph buffers and UI objects share the ESP-IDF heap instead of a fixed LVGL pool. +CONFIG_LV_USE_CLIB_MALLOC=y CONFIG_LV_USE_CLIB_STRING=y CONFIG_LV_USE_CLIB_SPRINTF=y CONFIG_LV_OBJ_STYLE_CACHE=y @@ -72,8 +74,17 @@ CONFIG_ESP_VIDEO_ENABLE_DVP_VIDEO_DEVICE=y CONFIG_CAMERA_OV3640=y CONFIG_CAMERA_OV3640_DVP_YUV422_UYVY_640X480_30FPS=y CONFIG_CAMERA_OV3640_DVP_DEFAULT_FMT_YUV422_UYVY_640X480_30FPS=y +# CameraBoards ship with either sensor, as in esp-mosaico-bsp mosaico_module_camera; +# esp_video probes both and keeps the one that answers. +CONFIG_CAMERA_SC101IOT=y +CONFIG_CAMERA_SC101IOT_AUTO_DETECT_DVP_INTERFACE_SENSOR=y +# SC101IOT VGA crops the sensor center; 720p retains the full field of view. +CONFIG_CAMERA_SC101IOT_DVP_YUV422_1280X720_15FPS=y +CONFIG_CAMERA_SC101IOT_DVP_DEFAULT_FMT_YUV422_1280X720_15FPS=y +# CONFIG_CAMERA_SC101IOT_DVP_YUV422_640X480_15FPS is not set +# CONFIG_CAMERA_SC101IOT_DVP_DEFAULT_FMT_YUV422_640X480_15FPS is not set CONFIG_MOSAICO_GESTURE_INPUT=y -CONFIG_MOSAICO_CAMERA_ROTATION=180 +CONFIG_MOSAICO_CAMERA_ROTATION=90 CONFIG_MOSAICO_CAMERA_MIRROR=y # The vision pipeline supplies RGB888; other ESP-DL converters are unused. diff --git a/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp b/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp index 8d9c54a..903d814 100644 --- a/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp +++ b/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp @@ -8,6 +8,8 @@ using namespace usage_panel::mosaico; namespace { +std::array image_output; + HandObservation hand(float x, float y = 0.5F, float ok = 0) { return {1, x, y, 0.25F, 0.35F, 0.9F, ok}; @@ -21,23 +23,31 @@ void arm(GestureTracker& tracker) TEST_CASE("swipes require release and issue exactly one directional command", "[gesture]") { - for (int direction : {-1, 1}) { - GestureTracker tracker; - arm(tracker); - TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F), 1500))); - TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F + direction * 0.15F), 1600))); - const auto expected = direction < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight; - TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(hand(0.5F + direction * 0.3F), 1700))); - for (unsigned t = 1800; t < 3000; t += 100) - TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F), t))); - tracker.update({}, 3000); - tracker.update(hand(0.5F), 3100); - TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.8F), 3300))); - tracker.update({}, 3400); - tracker.update({}, 3800); - tracker.update(hand(0.3F), 3900); - tracker.update(hand(0.45F), 4000); - TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.6F), 4100))); + for (bool vertical : {false, true}) { + for (int direction : {-1, 1}) { + const auto point = [vertical](float position) { + return vertical ? hand(0.5F, position) : hand(position); + }; + GestureTracker tracker; + arm(tracker); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(point(0.5F), 1500))); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(point(0.5F + direction * 0.15F), 1600))); + const auto expected = vertical + ? (direction < 0 ? GestureAction::SwipeUp : GestureAction::SwipeDown) + : (direction < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight); + TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(point(0.5F + direction * 0.3F), 1700))); + for (unsigned t = 1800; t < 3000; t += 100) + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(point(0.5F), t))); + tracker.update({}, 3000); + tracker.update(point(0.5F), 3100); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(point(0.8F), 3300))); + tracker.update({}, 3400); + tracker.update({}, 3800); + tracker.update(point(0.3F), 3900); + tracker.update(point(0.45F), 4000); + TEST_ASSERT_EQUAL_INT(int(vertical ? GestureAction::SwipeDown : GestureAction::SwipeRight), + int(tracker.update(point(0.6F), 4100))); + } } } @@ -54,19 +64,61 @@ TEST_CASE("OK requires a stable hold and release after context invalidation", "[ TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F, 0.5F, 0.95F), t))); } -TEST_CASE("vertical motion jitter ambiguous hands and stale samples do not trigger", "[gesture]") +TEST_CASE("single dropouts keep a swipe and an OK hold alive", "[gesture]") +{ + HandObservation weak = hand(0.6F); + weak.score = 0.55F; + for (const HandObservation& dropout : {HandObservation{}, weak}) { + GestureTracker tracker; + arm(tracker); + tracker.update(hand(0.5F), 1500); + tracker.update(dropout, 1600); + tracker.update(hand(0.65F), 1700); + TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.8F), 1800))); + } + + GestureTracker tracker; + arm(tracker); + tracker.update(hand(0.5F, 0.5F, 0.95F), 1500); + tracker.update(hand(0.5F, 0.5F, 0.95F), 1600); + tracker.update(hand(0.5F, 0.5F, 0.95F), 1700); + tracker.update({}, 1800); + TEST_ASSERT_EQUAL_INT(40, tracker.hold_progress()); + tracker.update(hand(0.5F, 0.5F, 0.95F), 1900); + TEST_ASSERT_EQUAL_INT(int(GestureAction::EnterClock), int(tracker.update(hand(0.5F, 0.5F, 0.95F), 2000))); +} + +TEST_CASE("slow frames and position jumps restart tracking without requiring release", "[gesture]") { - for (unsigned scenario = 0; scenario < 5; ++scenario) { + GestureTracker tracker; + arm(tracker); + tracker.update(hand(0.2F), 1500); + tracker.update(hand(0.2F), 1900); // 400 ms gap: restart here. + tracker.update(hand(0.35F), 2000); + TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.5F), 2100))); + + tracker.update({}, 2300); + tracker.update({}, 2700); + tracker.update(hand(0.9F), 2800); + tracker.update(hand(0.3F), 2900); // Jump: restart here. + tracker.update(hand(0.45F), 3000); + TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.6F), 3100))); +} + +TEST_CASE("diagonal motion jitter ambiguous hands and stale samples do not trigger", "[gesture]") +{ + for (unsigned scenario = 0; scenario < 6; ++scenario) { GestureTracker tracker; arm(tracker); for (unsigned i = 0; i < 7; ++i) { HandObservation observation = hand(0.2F + i * 0.08F); unsigned time = 1500 + i * 100; - if (scenario == 0) { observation.x = 0.5F; observation.y = 0.2F + i * 0.1F; } + if (scenario == 0) observation.y = observation.x; if (scenario == 1) observation.x = i % 2 ? 0.55F : 0.45F; if (scenario == 2) observation.count = 2; if (scenario == 3) time = 1500 + i * 400; if (scenario == 4) observation.x = std::numeric_limits::quiet_NaN(); + if (scenario == 5) { observation.x = 0.5F; observation.y = i % 2 ? 0.55F : 0.45F; } TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(observation, time))); } } @@ -76,11 +128,16 @@ TEST_CASE("wake consumes the first swipe and protected states reject all command { TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeLeft, true, true))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeRight, true, true))); - TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::SwipeLeft, true, false))); - TEST_ASSERT_EQUAL_INT(int(GestureEffect::Previous), int(route_gesture(GestureAction::SwipeRight, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Previous), int(route_gesture(GestureAction::SwipeLeft, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::SwipeRight, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeUp, true, true))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeDown, true, true))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Previous), int(route_gesture(GestureAction::SwipeUp, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::SwipeDown, true, false))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::Clock), int(route_gesture(GestureAction::EnterClock, true, false))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::None), int(route_gesture(GestureAction::EnterClock, true, true))); - for (auto action : {GestureAction::SwipeLeft, GestureAction::SwipeRight, GestureAction::EnterClock}) + for (auto action : {GestureAction::SwipeLeft, GestureAction::SwipeRight, GestureAction::SwipeUp, + GestureAction::SwipeDown, GestureAction::EnterClock}) TEST_ASSERT_EQUAL_INT(int(GestureEffect::None), int(route_gesture(action, false, false))); } @@ -88,12 +145,13 @@ TEST_CASE("UYVY conversion respects padded stride mirror and four rotations", "[ { // Four grayscale quadrants plus row padding make the geometric mapping observable. const uint8_t source[] = {128, 16, 128, 80, 255, 255, 128, 160, 128, 235, 255, 255}; - static std::array output; + auto& output = image_output; const unsigned expected[4][4] = {{0, 74, 167, 255}, {167, 0, 255, 74}, {255, 167, 74, 0}, {74, 255, 0, 167}}; for (unsigned rotation = 0; rotation < 4; ++rotation) { - const unsigned w = rotation & 1 ? kGestureImageHeight : kGestureImageWidth; - const unsigned h = rotation & 1 ? kGestureImageWidth : kGestureImageHeight; + const auto size = gesture_image_size(2, 2, rotation); + const unsigned w = size.width; + const unsigned h = size.height; TEST_ASSERT_TRUE(prepare_gesture_image(source, sizeof(source), 2, 2, 6, output.data(), rotation, false)); const unsigned pixels[] = {0, w - 1, (h - 1) * w, h * w - 1}; for (unsigned i = 0; i < 4; ++i) @@ -104,3 +162,47 @@ TEST_CASE("UYVY conversion respects padded stride mirror and four rotations", "[ TEST_ASSERT_FALSE(prepare_gesture_image(source, 4, 2, 2, 6, output.data(), 0, false)); TEST_ASSERT_FALSE(prepare_gesture_image(source, sizeof(source), 3, 2, 6, output.data(), 0, false)); } + +TEST_CASE("full sensor frame preserves aspect ratio and edges through rotation", "[gesture]") +{ + const auto hd = gesture_image_size(1280, 720, 0); + TEST_ASSERT_EQUAL_UINT(320, hd.width); + TEST_ASSERT_EQUAL_UINT(180, hd.height); + const auto vga = gesture_image_size(640, 480, 0); + TEST_ASSERT_EQUAL_UINT(320, vga.width); + TEST_ASSERT_EQUAL_UINT(240, vga.height); + TEST_ASSERT_EQUAL_UINT(0, gesture_image_size(0, 720, 0).width); + TEST_ASSERT_EQUAL_UINT(0, gesture_image_size(1280, 720, 4).width); + + // A 16:9 frame with distinct corners verifies that fitting does not crop edges. + constexpr unsigned stride = 36; + std::array source{}; + for (unsigned y = 0; y < 9; ++y) { + for (unsigned x = 0; x < 16; x += 2) { + const uint8_t luma = y < 4 ? (x < 8 ? 16 : 80) : (x < 8 ? 160 : 235); + const unsigned offset = y * stride + x * 2; + source[offset] = source[offset + 2] = 128; + source[offset + 1] = source[offset + 3] = luma; + } + } + auto& output = image_output; + const unsigned corners[2][4][4] = { + {{0, 74, 167, 255}, {167, 0, 255, 74}, {255, 167, 74, 0}, {74, 255, 0, 167}}, + {{74, 0, 255, 167}, {255, 74, 167, 0}, {167, 255, 0, 74}, {0, 167, 74, 255}} + }; + for (unsigned mirror = 0; mirror < 2; ++mirror) { + for (unsigned rotation = 0; rotation < 4; ++rotation) { + output.fill(42); + const auto size = gesture_image_size(16, 9, rotation); + TEST_ASSERT_EQUAL_UINT(rotation & 1 ? 180 : 320, size.width); + TEST_ASSERT_EQUAL_UINT(rotation & 1 ? 320 : 180, size.height); + TEST_ASSERT_TRUE(prepare_gesture_image(source.data(), source.size(), 16, 9, + stride, output.data(), rotation, mirror)); + const unsigned pixels[] = {0, size.width - 1, (size.height - 1) * size.width, + size.width * size.height - 1}; + for (unsigned i = 0; i < 4; ++i) + TEST_ASSERT_INT_WITHIN(1, corners[mirror][rotation][i], output[pixels[i] * 3]); + TEST_ASSERT_EQUAL_UINT8(42, output[size.width * size.height * 3]); + } + } +} From 3c2c6d3e0f1dbeba1ff1468370f8a5cc7fb10801 Mon Sep 17 00:00:00 2001 From: Wulu Date: Wed, 23 Sep 2026 16:38:53 +0800 Subject: [PATCH 04/15] fix(mosaico): persist panel settings atomically and stabilize dependency lock --- docs/design/mosaico-gesture-input.md | 4 +- docs/validation/mosaico-gestures.md | 2 +- firmware/targets/esp_mosaico/README.md | 6 +-- .../targets/esp_mosaico/dependencies.lock | 2 +- .../targets/esp_mosaico/main/CMakeLists.txt | 2 +- .../targets/esp_mosaico/main/app_main.cpp | 10 ++-- .../esp_mosaico/main/camera_setting.cpp | 31 ----------- .../esp_mosaico/main/camera_setting.hpp | 12 ----- .../esp_mosaico/main/idf_component.yml | 2 + .../esp_mosaico/main/panel_settings.cpp | 49 +++++++++++++++++ .../esp_mosaico/main/panel_settings.hpp | 20 +++++++ .../test_apps/logic/main/CMakeLists.txt | 4 +- .../logic/main/test_panel_settings.cpp | 52 +++++++++++++++++++ 13 files changed, 139 insertions(+), 57 deletions(-) delete mode 100644 firmware/targets/esp_mosaico/main/camera_setting.cpp delete mode 100644 firmware/targets/esp_mosaico/main/camera_setting.hpp create mode 100644 firmware/targets/esp_mosaico/main/panel_settings.cpp create mode 100644 firmware/targets/esp_mosaico/main/panel_settings.hpp create mode 100644 firmware/targets/esp_mosaico/test_apps/logic/main/test_panel_settings.cpp diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md index 12a635b..68335e6 100644 --- a/docs/design/mosaico-gesture-input.md +++ b/docs/design/mosaico-gesture-input.md @@ -40,7 +40,7 @@ | ESP-WHO | README 描述采集/推理异步运行；板卡清单未列 Mosaico | 仅作生命周期与任务划分参考 | | 当前 BSP | `BSP_CAPS_CAMERA` 为 0，已有扩展槽电源/I2C/GPIO 基础，无采集组件依赖 | CameraBoard 接入是首个工程任务，不能只改能力宏 | | 摄像头参考实现 | 相邻 esp-mosaico-claw 工程有 OV3640 DVP 的 `mosaico_camera`，当前接口支持左槽和帧获取/归还 | 它依赖 `subboard_support`、模块管理、board manager 及 esp-claw 内的 esp_video；不能直接复制一个组件就认为可用 | -| 固件存储 | 目标配置 16 MB Flash；原有两个 OTA app 分区各 4 MiB；本实现改为 7 MiB | 完整应用约 5.16 MiB，首次需有线迁移，见下文 | +| 固件存储 | 目标配置 16 MB Flash；两个 OTA app 分区各 7 MiB | 完整应用约 5.16 MiB，模型随应用更新 | 相邻工程核对时 HEAD 为 `903fe11ec3ca4a8881afef104e53a8da040896ac`，摄像头头文件和实现存在未提交修改，README 的默认分辨率/缓冲数量也与当前头文件不同。它只提供参考，实施时必须选定可复现的来源和许可，不能将本地工作区当作已验证的上游版本。 @@ -105,7 +105,7 @@ CameraBoard / esp_video - [ ] 先运行静态图 HandDetect，再接实时帧；确认推理结果和帧归还；在现有 UI、BLE、IMU、电量轮询共存时采集耗时和内存。 - [ ] 核算最终 app 大小、模型大小、内部 RAM/PSRAM 低水位及连续块。224×224 RGB888 输入约 147 KiB；640×480 双缓冲 RGB565 约 1.17 MiB，仅是缓冲示例，不含张量、显示、驱动和栈，也不表示该格式已被硬件接受。 -存储决策：模型内嵌 app，使模型随 A/B OTA 和回滚一起切换。裁减未使用的 ESP-DL 像素转换后，完整应用仍约 5.16 MiB，超过旧 4 MiB 槽；采用 7 MiB 双槽，保留 NVS、PHY 与 otadata 地址。首次必须分段有线写入新分区表、应用及初始 OTA 元数据；旧 OTA 不能完成此迁移，旧第二槽不能作为迁移后的回滚来源。具体地址和恢复步骤见[首次烧录与验收](../validation/mosaico-gestures.md)。 +存储决策：模型内嵌 app，使模型随 A/B OTA 和回滚一起切换。裁减未使用的 ESP-DL 像素转换后，完整应用约 5.16 MiB，采用 7 MiB 双槽。NVS、PHY 与 otadata 使用当前分区表指定的地址。设备使用该分区表后，可通过 OTA 更新应用与模型；完整安装和实机验证见[固件安装与验收](../validation/mosaico-gestures.md)。 **完成条件：**可复现的依赖与构建、可用的实时 bbox、资源与延迟记录、正常停止/缺摄像头降级。只构建成功不算实时识别通过。当前已完成软件接入及 UI，以便一次烧录完成端到端验收；实时识别和资源指标未通过实机验证前，不视为产品验收完成。 diff --git a/docs/validation/mosaico-gestures.md b/docs/validation/mosaico-gestures.md index 6c320e2..be2ca1d 100644 --- a/docs/validation/mosaico-gestures.md +++ b/docs/validation/mosaico-gestures.md @@ -1,4 +1,4 @@ -# Mosaico 手势固件：首次烧录与验收 +# Mosaico 手势固件：安装与验收 本功能在设备本地运行：左挥或上挥上一页、右挥或下挥下一页；时钟中首次挥手只恢复原页面；👌 OK 稳定停留约 500 ms 进入时钟。一轮操作最多触发一次，需要手离开后再操作。触摸和 AI 键继续可用。 diff --git a/firmware/targets/esp_mosaico/README.md b/firmware/targets/esp_mosaico/README.md index 5c51d98..6281f6d 100644 --- a/firmware/targets/esp_mosaico/README.md +++ b/firmware/targets/esp_mosaico/README.md @@ -40,9 +40,9 @@ eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 后续开发入口：[参考仓库、固定版本与代码导航](CAMERA_UPSTREAM.md#repository-reference-map)；[交互设计与实施记录](../../../docs/design/mosaico-gesture-input.md)。 -左槽 CameraBoard（OV3640 或 SC101IOT，自动识别）可选开启本地手势：左挥或上挥上一页、右挥或下挥下一页，时钟中首挥只恢复原页；👌 OK 稳定保持约 500 ms 进入时钟。设置中的 `CAMERA` 开关默认关闭，切换后需 `SAVE`；开关打开时顶栏显示摄像头标识。Type-C 口是 USB OTG，只在 ROM 下载模式下枚举，应用日志只从 UART0（右侧排针 H1 的 15 脚 TX0 / GPIO58，或底板背面调试焊盘 TX）输出。图像仅在本机处理。 +左槽 CameraBoard（OV3640 或 SC101IOT，自动识别）可选开启本地手势：左挥或上挥上一页、右挥或下挥下一页，时钟中首挥只恢复原页；👌 OK 稳定保持约 500 ms 进入时钟。设置中的 `CAMERA` 开关默认关闭，切换后需 `SAVE`；开关打开时顶栏显示摄像头标识。亮度、自动时钟超时和相机开关作为一条 NVS 记录保存；记录缺失或无效时使用默认显示设置并关闭相机。Type-C 口是 USB OTG，只在 ROM 下载模式下枚举，应用日志只从 UART0（右侧排针 H1 的 15 脚 TX0 / GPIO58，或底板背面调试焊盘 TX）输出。图像仅在本机处理。 -模型与代码内嵌同一应用，首次从旧 4 MiB 分区升级必须有线迁移。保留设置时使用分段烧录，不能用填充后的完整镜像替代。依赖来源见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)，安装、恢复与实机验证见[手势验收指南](../../../docs/validation/mosaico-gestures.md)。当前仅通过软件测试和构建；相机方向、识别率、性能、功耗和提示布局仍待设备验收。 +模型与代码内嵌同一应用，在当前 7 MiB 双槽分区中共同进行 OTA 更新与回滚。有线更新应用并保留设置时使用分段烧录，不能用填充后的完整镜像替代。依赖来源见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)，安装、恢复与实机验证见[手势验收指南](../../../docs/validation/mosaico-gestures.md)。当前仅通过软件测试和构建；相机方向、识别率、性能、功耗和提示布局仍待设备验收。 ### 显示刷新 @@ -74,7 +74,7 @@ TE 信号缺失时最多等待 25 ms 后继续刷新，并在超时、恢复状 eim run "idf.py -C firmware/targets/esp_mosaico/test_apps/logic build" v6.1 ``` -覆盖本目标独有的 `local_clock`、`orientation`、手势轨迹/动作路由和图像坐标转换。共享面板行为由 `firmware/test_apps/panel_state` 覆盖。没有实机时该命令只证明测试镜像编译通过。 +覆盖本目标独有的 `local_clock`、`orientation`、手势轨迹/动作路由、图像坐标转换和设置持久化。共享面板行为由 `firmware/test_apps/panel_state` 覆盖。没有实机时该命令只证明测试镜像编译通过。 ## 后续步骤 diff --git a/firmware/targets/esp_mosaico/dependencies.lock b/firmware/targets/esp_mosaico/dependencies.lock index 570952e..ef2fab9 100644 --- a/firmware/targets/esp_mosaico/dependencies.lock +++ b/firmware/targets/esp_mosaico/dependencies.lock @@ -578,6 +578,6 @@ direct_dependencies: - idf - lvgl/lvgl - tangyumei3535/bmm150_sensorapi -manifest_hash: e9597de848f4dedfd77d40c2df27a3229221ce044985973fb9e4d7ee9873d515 +manifest_hash: 53e3bd813b8217d5baee702eb587a7358830bd13a6fd85c8f6f41b0109f559f2 target: esp32s31 version: 3.0.0 diff --git a/firmware/targets/esp_mosaico/main/CMakeLists.txt b/firmware/targets/esp_mosaico/main/CMakeLists.txt index c4494e9..ae8aa2b 100644 --- a/firmware/targets/esp_mosaico/main/CMakeLists.txt +++ b/firmware/targets/esp_mosaico/main/CMakeLists.txt @@ -5,7 +5,7 @@ idf_component_register( "local_clock.cpp" "orientation.cpp" "power_monitor.cpp" - "camera_setting.cpp" + "panel_settings.cpp" "gesture_input.cpp" "gesture_tracker.cpp" INCLUDE_DIRS "." diff --git a/firmware/targets/esp_mosaico/main/app_main.cpp b/firmware/targets/esp_mosaico/main/app_main.cpp index 1ca86d9..7d19326 100644 --- a/firmware/targets/esp_mosaico/main/app_main.cpp +++ b/firmware/targets/esp_mosaico/main/app_main.cpp @@ -5,7 +5,7 @@ #include "display_ui.hpp" #include "local_clock.hpp" #include "gesture_input.hpp" -#include "camera_setting.hpp" +#include "panel_settings.hpp" #include "usage_panel_state/panel_presentation.hpp" #include "power_monitor.hpp" #include "usage_panel_state/render_schedule.hpp" @@ -194,14 +194,15 @@ extern "C" void app_main(void) } const bool ai_button_ready = init_ai_button(); - DisplaySettings saved_settings = load_display_settings({ + const PanelSettings persisted_settings = load_panel_settings({ CONFIG_MOSAICO_USAGE_PANEL_DISPLAY_BRIGHTNESS_PERCENT, CONFIG_MOSAICO_USAGE_PANEL_SCREENSAVER_TIMEOUT_SECONDS}); + DisplaySettings saved_settings = persisted_settings.display; DisplaySettings settings = saved_settings; bool settings_open = false; // camera_enabled is persisted and drives capture; camera_setting is the // settings-page preview, committed by SAVE together with the display values. - bool camera_enabled = load_camera_enabled(); + bool camera_enabled = persisted_settings.camera_enabled; bool camera_setting = camera_enabled; #if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW gestures.set_preview_sink( @@ -328,8 +329,7 @@ extern "C" void app_main(void) else if (action >= SettingsAction::TimeoutOff && action <= SettingsAction::Timeout30) select_clock_timeout(action, settings.clock_timeout_seconds); else if (action == SettingsAction::Save) { - esp_err_t error = save_display_settings(settings); - if (error == ESP_OK) error = save_camera_enabled(camera_setting); + const esp_err_t error = save_panel_settings({settings, camera_setting}); save_error = error != ESP_OK; if (!save_error) { saved_settings = settings; diff --git a/firmware/targets/esp_mosaico/main/camera_setting.cpp b/firmware/targets/esp_mosaico/main/camera_setting.cpp deleted file mode 100644 index 4781f4d..0000000 --- a/firmware/targets/esp_mosaico/main/camera_setting.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "camera_setting.hpp" -#include "nvs.h" - -namespace usage_panel::mosaico { -namespace { -constexpr char kNamespace[] = "camera"; -constexpr char kEnabledKey[] = "enabled"; -} - -bool load_camera_enabled() -{ - nvs_handle_t handle; - if (nvs_open(kNamespace, NVS_READONLY, &handle) != ESP_OK) return false; - uint8_t enabled = 0; - const bool stored = nvs_get_u8(handle, kEnabledKey, &enabled) == ESP_OK && enabled <= 1; - nvs_close(handle); - return stored && enabled; -} - -esp_err_t save_camera_enabled(bool enabled) -{ - nvs_handle_t handle; - esp_err_t error = nvs_open(kNamespace, NVS_READWRITE, &handle); - if (error != ESP_OK) return error; - error = nvs_set_u8(handle, kEnabledKey, enabled ? 1 : 0); - if (error == ESP_OK) error = nvs_commit(handle); - nvs_close(handle); - return error; -} - -} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/camera_setting.hpp b/firmware/targets/esp_mosaico/main/camera_setting.hpp deleted file mode 100644 index cd530f1..0000000 --- a/firmware/targets/esp_mosaico/main/camera_setting.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include "esp_err.h" - -namespace usage_panel::mosaico { - -/** Returns the persisted camera switch; absent or invalid storage reads as off. */ -bool load_camera_enabled(); -/** Persists the camera switch; called by settings SAVE together with the display values. */ -esp_err_t save_camera_enabled(bool enabled); - -} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/idf_component.yml b/firmware/targets/esp_mosaico/main/idf_component.yml index f675b62..1e6fc52 100644 --- a/firmware/targets/esp_mosaico/main/idf_component.yml +++ b/firmware/targets/esp_mosaico/main/idf_component.yml @@ -4,6 +4,8 @@ dependencies: # Keeping both dependencies in the first pass preserves the locked graph. espressif/esp_lvgl_adapter: "0.6.2" lvgl/lvgl: "^9.5.0,<9.6.0" + # The display adapter is required before BSP sdkconfig rules can be evaluated. + espressif/esp_lvgl_adapter: "0.6.2" espressif/esp_video: "2.5.0" espressif/hand_detect: "0.2.0" espressif/hand_gesture_recognition: "0.2.0" diff --git a/firmware/targets/esp_mosaico/main/panel_settings.cpp b/firmware/targets/esp_mosaico/main/panel_settings.cpp new file mode 100644 index 0000000..f48e4db --- /dev/null +++ b/firmware/targets/esp_mosaico/main/panel_settings.cpp @@ -0,0 +1,49 @@ +#include "panel_settings.hpp" +#include "nvs.h" + +namespace usage_panel::mosaico { +namespace { +constexpr char kNamespace[] = "mosaico"; +constexpr char kSettingsKey[] = "settings"; +// One NVS integer keeps display and capture preferences in the same storage update. +// Bits 0..6 hold brightness, bit 7 enables capture, and bits 8..31 hold timeout seconds. +constexpr uint32_t kBrightnessMask = 0x7f; +constexpr uint32_t kCameraMask = 0x80; +constexpr unsigned kTimeoutShift = 8; + +bool valid(const DisplaySettings& settings) +{ + return settings.brightness >= 1 && settings.brightness <= 100 && + settings.clock_timeout_seconds <= 86400; +} +} + +PanelSettings load_panel_settings(DisplaySettings defaults) +{ + nvs_handle_t handle; + if (nvs_open(kNamespace, NVS_READONLY, &handle) != ESP_OK) return {defaults}; + uint32_t packed = 0; + const esp_err_t error = nvs_get_u32(handle, kSettingsKey, &packed); + nvs_close(handle); + const DisplaySettings display{static_cast(packed & kBrightnessMask), + packed >> kTimeoutShift}; + if (error != ESP_OK || !valid(display)) return {defaults}; + return {display, (packed & kCameraMask) != 0}; +} + +esp_err_t save_panel_settings(const PanelSettings& settings) +{ + if (!valid(settings.display)) return ESP_ERR_INVALID_ARG; + const uint32_t packed = settings.display.brightness | + (settings.camera_enabled ? kCameraMask : 0) | + (settings.display.clock_timeout_seconds << kTimeoutShift); + nvs_handle_t handle; + esp_err_t error = nvs_open(kNamespace, NVS_READWRITE, &handle); + if (error != ESP_OK) return error; + error = nvs_set_u32(handle, kSettingsKey, packed); + if (error == ESP_OK) error = nvs_commit(handle); + nvs_close(handle); + return error; +} + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/main/panel_settings.hpp b/firmware/targets/esp_mosaico/main/panel_settings.hpp new file mode 100644 index 0000000..b0c06c7 --- /dev/null +++ b/firmware/targets/esp_mosaico/main/panel_settings.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "usage_panel_state/display_settings.hpp" + +namespace usage_panel::mosaico { + +struct PanelSettings { + DisplaySettings display; + bool camera_enabled = false; +}; + +/** Loads one settings record; missing or invalid records retain defaults and disable capture. */ +PanelSettings load_panel_settings(DisplaySettings defaults); +/** + * Saves all fields in one NVS value. Brightness must be 1..100 and timeout 0..86400 seconds. + * Returns ESP_ERR_INVALID_ARG for invalid display values, otherwise the NVS operation status. + */ +esp_err_t save_panel_settings(const PanelSettings& settings); + +} // namespace usage_panel::mosaico diff --git a/firmware/targets/esp_mosaico/test_apps/logic/main/CMakeLists.txt b/firmware/targets/esp_mosaico/test_apps/logic/main/CMakeLists.txt index e658d57..8539caa 100644 --- a/firmware/targets/esp_mosaico/test_apps/logic/main/CMakeLists.txt +++ b/firmware/targets/esp_mosaico/test_apps/logic/main/CMakeLists.txt @@ -4,12 +4,14 @@ idf_component_register( "test_local_clock.cpp" "test_orientation.cpp" "test_gestures.cpp" + "test_panel_settings.cpp" + "../../../main/panel_settings.cpp" "../../../main/local_clock.cpp" "../../../main/orientation.cpp" "../../../main/gesture_tracker.cpp" "../../../main/gesture_image.cpp" INCLUDE_DIRS "../../../main" - REQUIRES unity usage_protocol usage_panel_state + REQUIRES unity usage_protocol usage_panel_state nvs_flash WHOLE_ARCHIVE ) target_compile_features(${COMPONENT_LIB} PUBLIC cxx_std_17) diff --git a/firmware/targets/esp_mosaico/test_apps/logic/main/test_panel_settings.cpp b/firmware/targets/esp_mosaico/test_apps/logic/main/test_panel_settings.cpp new file mode 100644 index 0000000..9f70003 --- /dev/null +++ b/firmware/targets/esp_mosaico/test_apps/logic/main/test_panel_settings.cpp @@ -0,0 +1,52 @@ +#include "panel_settings.hpp" +#include "nvs.h" +#include "nvs_flash.h" +#include "unity.h" + +#include + +using namespace usage_panel; +using namespace usage_panel::mosaico; + +TEST_CASE("panel preferences persist together and reject invalid updates", "[settings]") +{ + TEST_ASSERT_EQUAL(ESP_OK, nvs_flash_init()); + for (bool camera : {false, true}) { + const PanelSettings expected{{camera ? uint8_t{100} : uint8_t{1}, + camera ? 86400U : 0U}, camera}; + TEST_ASSERT_EQUAL(ESP_OK, save_panel_settings(expected)); + TEST_ASSERT_EQUAL(ESP_ERR_INVALID_ARG, save_panel_settings({{0, 60}, !camera})); + TEST_ASSERT_EQUAL(ESP_ERR_INVALID_ARG, save_panel_settings({{101, 60}, !camera})); + TEST_ASSERT_EQUAL(ESP_ERR_INVALID_ARG, save_panel_settings({{50, 86401}, !camera})); + TEST_ASSERT_EQUAL(ESP_OK, nvs_flash_deinit()); + TEST_ASSERT_EQUAL(ESP_ERR_NVS_NOT_INITIALIZED, save_panel_settings({{50, 60}, !camera})); + TEST_ASSERT_EQUAL(ESP_OK, nvs_flash_init()); + const auto actual = load_panel_settings({50, 300}); + TEST_ASSERT_EQUAL(expected.display.brightness, actual.display.brightness); + TEST_ASSERT_EQUAL(expected.display.clock_timeout_seconds, actual.display.clock_timeout_seconds); + TEST_ASSERT_EQUAL(expected.camera_enabled, actual.camera_enabled); + } +} + +TEST_CASE("missing or invalid panel records use defaults with capture off", "[settings]") +{ + TEST_ASSERT_EQUAL(ESP_OK, nvs_flash_init()); + nvs_handle_t handle; + TEST_ASSERT_EQUAL(ESP_OK, nvs_open("mosaico", NVS_READWRITE, &handle)); + // Invalid brightness and timeout must reject the entire record, including capture enablement. + for (uint32_t packed : {uint32_t{0x80}, uint32_t{(86401U << 8) | 0x80 | 50}}) { + TEST_ASSERT_EQUAL(ESP_OK, nvs_set_u32(handle, "settings", packed)); + TEST_ASSERT_EQUAL(ESP_OK, nvs_commit(handle)); + const auto actual = load_panel_settings({40, 600}); + TEST_ASSERT_EQUAL(40, actual.display.brightness); + TEST_ASSERT_EQUAL(600, actual.display.clock_timeout_seconds); + TEST_ASSERT_FALSE(actual.camera_enabled); + } + TEST_ASSERT_EQUAL(ESP_OK, nvs_erase_key(handle, "settings")); + TEST_ASSERT_EQUAL(ESP_OK, nvs_commit(handle)); + nvs_close(handle); + const auto actual = load_panel_settings({30, 60}); + TEST_ASSERT_EQUAL(30, actual.display.brightness); + TEST_ASSERT_EQUAL(60, actual.display.clock_timeout_seconds); + TEST_ASSERT_FALSE(actual.camera_enabled); +} From 9db999a7589761b6a5297e1f58dd04b2bef5a08c Mon Sep 17 00:00:00 2001 From: Wulu Date: Wed, 23 Sep 2026 16:56:27 +0800 Subject: [PATCH 05/15] fix(mosaico): refine camera preview and skip hidden frame locking --- docs/design/amoled-ui.md | 1 + docs/design/mosaico-gesture-input.md | 2 +- docs/validation/mosaico-gestures.md | 4 ++-- firmware/targets/esp_mosaico/README.md | 4 ++-- .../esp_mosaico/main/Kconfig.projbuild | 8 ++++---- .../targets/esp_mosaico/main/app_main.cpp | 2 +- .../targets/esp_mosaico/main/display_ui.cpp | 20 +++++++++---------- .../targets/esp_mosaico/main/display_ui.hpp | 3 +-- .../esp_mosaico/main/gesture_input.cpp | 2 +- .../esp_mosaico/main/gesture_input.hpp | 2 +- 10 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docs/design/amoled-ui.md b/docs/design/amoled-ui.md index ce08be2..2b3a8c7 100644 --- a/docs/design/amoled-ui.md +++ b/docs/design/amoled-ui.md @@ -73,6 +73,7 @@ Waveshare：屏幕边距 24px；卡片宽 432px；卡片内边距 20–22px； - **连接状态**：右上角一个 8px 状态色圆点 + 14px `#C7CCD4` 文字。不显示 `SECURE` 之类无操作价值的标签。 - **电池**：连接状态右侧，30×14 描边（1px `#565D68`）+ 2px 正极点，内含 4 段。**分级显示，不显示百分比**：FULL=4 段、HIGH=3 段、MID=2 段、LOW=1 段、无数据=0 段；点亮色 `#C7CCD4`，熄灭段 `#2C313B`；仅 LOW 时点亮段转 `#F5B84C`。**充电**：外框左侧 gap 5px 有 7×12 闪电槽，**始终占位**（不充电时留空，绝不收起，否则插拔 USB 会让 `LINKED` 平移）；充电时填 `#C7CCD4`，段数照常反映电量，不做动画；充电时 LOW 不再变黄（外部供电已接入，此时低电量告警不再提供操作价值），屏保 `LOW` 角标同时撤掉。USB 只是供电，不改变 `LINKED`，界面不出现 `USB` 标签。 - **摄像头标识**（仅 Mosaico）：`CAMERA` 开关已保存为打开时，连接状态点左侧 10px 显示 26×16 实心相机剪影（顶部 8×4 取景凸起，bg 色镜头环内含 4px 镜头点），与状态点垂直居中。采集中（启动或识别）为 `text` 色；开关打开但采集暂停（设置、配对、OTA）为 `dim`；采集失败时为 `dim` 并叠加 `#C7CCD4` 斜杠（外圈 bg 色描边）。开关关闭时不占位。顶栏不显示手势文字提示，仅保留 OK 停留进度条。保持灰阶，不使用 good/warn/bad 色。屏保不显示。 +- **相机预览**（仅 Mosaico）：识别就绪且处于普通页面时，点击顶栏摄像头标识切换预览，再次点击关闭；每次启动默认隐藏，不持久化。浮层位于内容区右上角，距右侧 24px、距内容区顶部 8px；按画面比例显示，最长边 160px，不裁切、不拉伸。只显示相机图像及 2px `text` 色手部定位框，无手时隐藏定位框，不叠加检测数量、置信度或状态文字。开关只控制预览显示，手势识别由设置页 CAMERA 控制。时钟、设置、配对、OTA、相机暂停或故障时隐藏浮层；识别恢复且返回普通页面后恢复本次运行的预览选择。 - **时间**：顶栏左侧，22px 700，`text` 色，24 小时制不显示秒。页面归属由底部导航指示，无需重复标题。硬件无温湿度传感器，界面不出现环境信息。 - **屏保态（SCREENSAVER）**：空闲 5 分钟后进入，触摸屏幕或晃动（IMU）唤醒，回到进入前的页面。规则是**整体降一级**——时钟 `muted #8B93A1`（120px，居中 top 158）、日期一行 20px `dim`、顶栏与底部导航全部隐藏、卡片去掉 panel 底与描边（纯 `bg`，AMOLED 省电）。屏幕上不得出现 `text` 级亮度。底部 13px `#2C313B` 提示 `TAP OR SHAKE TO WAKE`，进入 3 秒后淡出。仅电池 LOW 时右上角显示降亮的 `LOW` + 电池图标（`#8A6A2E`），其余状态一概不显示。防烧屏：整块内容每 60 秒随机位移 ±10px。 diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md index 68335e6..47e6468 100644 --- a/docs/design/mosaico-gesture-input.md +++ b/docs/design/mosaico-gesture-input.md @@ -162,7 +162,7 @@ eim run "idf.py -C firmware/test_apps/panel_state build" v6.1 - 单手轨迹、OK 500 ms 停留、离开后再触发、有界事件与上下文失效；主循环实现翻页、只唤醒、幂等进入时钟及受保护状态屏蔽。 - 设置页独立 `CAMERA` 开关默认关闭，与显示设置共用保存/取消；顶栏摄像头标识区分采集中、暂停与失败，不显示文字提示；停用释放采集与模型，手动重试。ESP-DL 内部部分分配使用断言；资源预检查降低风险，但不保证所有运行时内存不足均能无重启降级。 - 7 MiB 双 OTA 分区、Folder Push 上限和 Bridge 容量配置；无共享模型分区。 -- 手势开/关固件、Mosaico logic、共享 panel_state 测试镜像以及 AMOLED 2.16 固件构建通过。31 个手势/屏保主机 Unity 测试通过 ASan/UBSan；Bridge 620 项测试通过，7 项跳过。测试镜像尚未在设备运行。 +- 手势开/关固件、Mosaico logic、共享 panel_state 测试镜像以及 AMOLED 2.16 固件构建通过。已完成手势/屏保主机 Unity 测试的 ASan/UBSan 检查及 Bridge 自动化测试。软件与硬件验证范围见[验证范围](../verification.md)，新增用例的执行情况以[手势验收记录](../validation/mosaico-gestures.md)为准；测试镜像尚未在设备运行。 下一步是按[首次烧录与实机验收](../validation/mosaico-gestures.md)安装并开启手势，检查摄像头方向、真实模型输出、UI 提示布局、推理耗时、误触、启停与功耗。未烧录、未发布，也未验证物理 A/B 回滚。不能把本次软件验证解释为 P0/P1/P2 的硬件验收通过。 diff --git a/docs/validation/mosaico-gestures.md b/docs/validation/mosaico-gestures.md index be2ca1d..3594dfe 100644 --- a/docs/validation/mosaico-gestures.md +++ b/docs/validation/mosaico-gestures.md @@ -75,14 +75,12 @@ SC101IOT 使用完整 1280×720 画面，避免 VGA 模式的中心裁切；等 本版应用与 logic 测试镜像均已通过 ESP-IDF v6.1 构建，新增图像比例、旋转、镜像和边缘保留用例仅完成编译，尚未运行。应用已按六段写入设备，全部通过设备哈希校验；未写入 NVS、分区表或 OTA 元数据。重启日志确认运行镜像与产物匹配，并成功采集 1280×720 UYVY；观察窗口内无看门狗或采集失败，记录到一次右挥翻页。稳定采样的平均处理周期约 105～123 ms，PSRAM 空闲约 4.39 MB。用户反馈近距离取景有所改善，但旋转后的竖幅横向空间不足；具体距离、OK 识别及左右挥手触发率尚未验收。检查启动日志包含 `camera UYVY 1280x720`，预览应包含更完整的场景且手形无拉伸；分别在约 40、30、25 cm 检查完整入镜、OK 进入时钟与左右挥手。记录实际识别率、平均周期、帧超时和内存余量，确认反复关闭/开启 CAMERA 与时钟切换正常。25～30 cm 是测试目标，不是已验证的识别距离。 - ## 空中挥手的页面方向 空中手势按导航项顺序选择页面：右挥或下挥为下一页，左挥或上挥为上一页，末尾按现有页面顺序循环。触摸滑动语义保持不变；时钟中首次挥手仍只唤醒。前述日志中的 SwipeRight/Previous 等记录属于调整前版本，不作为当前映射的验收证据。 方向调整后的应用与 logic 测试镜像均通过 ESP-IDF v6.1 构建；路由测试尚未运行，方向修正已随四向手势应用烧录，方向一致性待实机确认。 - ## 上下挥手 上下挥手与左右挥手共用单手置信度、150～900 ms 动作窗口、至少三帧、25% 主轴净位移和释放确认。斜向轨迹及往返抖动不应翻页；时钟中任一方向首次挥动只恢复原页。实机需分别验证上挥上一页、下挥下一页，以及一次动作不会重复翻页；新增四向逻辑尚未实机验收。 @@ -93,6 +91,8 @@ SC101IOT 使用完整 1280×720 画面，避免 VGA 模式的中心裁切；等 预览默认隐藏，点击顶部相机图标切换显示；隐藏时跳过预览绘制，手势识别继续运行。应用通过 ESP-IDF v6.1 构建，已按六段烧录并全部通过设备哈希校验。仅更新应用分区，重启后已采集到分阶段性能日志，图标点击行为仍待用户确认。本版本包含采集转换、检测、分类和预览反馈的分阶段耗时统计。 +预览交互验收需检查：启动隐藏、点击图标开关、图像比例及手部定位框、无调试文字；进入时钟或受保护页面后隐藏，恢复识别并返回普通页面后恢复本次选择。隐藏期间应跳过预览显示锁和绘制；实际锁竞争与帧率变化仍需测量，不能以构建结果代替性能验收。 + ## 性能基线与后续优化（2026-09-23） 预览关闭且未执行 OK 分类的三个连续约 10 秒窗口中，每窗口处理 80 帧，约 8 fps。平均采集与转换耗时 18.6 ms，手掌检测（含前后处理）86.8 ms，反馈约 0.006 ms，循环另有固定 20 ms 等待，总帧间隔约 125 ms。`mean_cycle_ms` 不含反馈及固定等待，不能直接取倒数作为识别帧率。该窗口不代表正常页检测到手后执行 OK 分类的完整负载。 diff --git a/firmware/targets/esp_mosaico/README.md b/firmware/targets/esp_mosaico/README.md index 6281f6d..30b21b3 100644 --- a/firmware/targets/esp_mosaico/README.md +++ b/firmware/targets/esp_mosaico/README.md @@ -42,6 +42,8 @@ eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 左槽 CameraBoard（OV3640 或 SC101IOT，自动识别）可选开启本地手势：左挥或上挥上一页、右挥或下挥下一页，时钟中首挥只恢复原页；👌 OK 稳定保持约 500 ms 进入时钟。设置中的 `CAMERA` 开关默认关闭，切换后需 `SAVE`；开关打开时顶栏显示摄像头标识。亮度、自动时钟超时和相机开关作为一条 NVS 记录保存；记录缺失或无效时使用默认显示设置并关闭相机。Type-C 口是 USB OTG，只在 ROM 下载模式下枚举，应用日志只从 UART0（右侧排针 H1 的 15 脚 TX0 / GPIO58，或底板背面调试焊盘 TX）输出。图像仅在本机处理。 +点击顶部相机图标可显示或隐藏实时预览；预览默认隐藏，重启后恢复隐藏。关闭预览不影响手势识别，摄像头采集仍由设置页 CAMERA 开关控制。时钟、设置及受保护界面不显示预览。预览保留原始画面比例，只显示图像和手部定位框。返回可识别的普通页面后，恢复本次运行中选择的预览状态。 + 模型与代码内嵌同一应用，在当前 7 MiB 双槽分区中共同进行 OTA 更新与回滚。有线更新应用并保留设置时使用分段烧录，不能用填充后的完整镜像替代。依赖来源见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)，安装、恢复与实机验证见[手势验收指南](../../../docs/validation/mosaico-gestures.md)。当前仅通过软件测试和构建；相机方向、识别率、性能、功耗和提示布局仍待设备验收。 ### 显示刷新 @@ -80,5 +82,3 @@ eim run "idf.py -C firmware/targets/esp_mosaico/test_apps/logic build" v6.1 1. 实机验证触控方向、AI 键，以及 OTA 断电恢复与回滚； 2. 按实测结果校准 BQ27220 的外部供电判据。 - -点击顶部相机图标可显示或隐藏实时预览；预览默认隐藏，重启后恢复隐藏。关闭预览不影响手势识别，摄像头采集仍由设置页 CAMERA 开关控制。时钟、设置及受保护界面不显示预览。 diff --git a/firmware/targets/esp_mosaico/main/Kconfig.projbuild b/firmware/targets/esp_mosaico/main/Kconfig.projbuild index 569090a..af2b254 100644 --- a/firmware/targets/esp_mosaico/main/Kconfig.projbuild +++ b/firmware/targets/esp_mosaico/main/Kconfig.projbuild @@ -117,14 +117,14 @@ config MOSAICO_CAMERA_MIRROR help Mirror the front-facing camera into the user's screen coordinates. -config MOSAICO_GESTURE_DEBUG_PREVIEW +config MOSAICO_CAMERA_PREVIEW bool "Enable camera preview toggled by the status icon" default y depends on MOSAICO_GESTURE_INPUT help - The status-bar camera icon toggles a preview of the model input, - detection box and confidence scores. The preview starts hidden and - does not change whether gesture recognition is enabled. + The status-bar camera icon toggles a preview of the model input + and hand location box. The preview starts hidden, its visibility is + not persisted, and toggling it does not change gesture recognition. endmenu diff --git a/firmware/targets/esp_mosaico/main/app_main.cpp b/firmware/targets/esp_mosaico/main/app_main.cpp index 7d19326..4dde6fd 100644 --- a/firmware/targets/esp_mosaico/main/app_main.cpp +++ b/firmware/targets/esp_mosaico/main/app_main.cpp @@ -204,7 +204,7 @@ extern "C" void app_main(void) // settings-page preview, committed by SAVE together with the display values. bool camera_enabled = persisted_settings.camera_enabled; bool camera_setting = camera_enabled; -#if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW +#if CONFIG_MOSAICO_CAMERA_PREVIEW gestures.set_preview_sink( [](void* context, const uint8_t* rgb, unsigned width, unsigned height, const HandObservation& hand) { static_cast(context)->show_camera_preview(rgb, width, height, hand); diff --git a/firmware/targets/esp_mosaico/main/display_ui.cpp b/firmware/targets/esp_mosaico/main/display_ui.cpp index e451b7f..4696d65 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.cpp +++ b/firmware/targets/esp_mosaico/main/display_ui.cpp @@ -491,7 +491,7 @@ void DisplayUi::build_header(lv_obj_t* screen) // lens ring and a state-coloured lens centre. static const lv_point_precise_t kSlash[] = {{3, 2}, {23, 26}}; camera_indicator_ = make_shape(screen, 275, 13, 26, 28, 0); -#if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW +#if CONFIG_MOSAICO_CAMERA_PREVIEW lv_obj_add_flag(camera_indicator_, LV_OBJ_FLAG_CLICKABLE); lv_obj_set_ext_click_area(camera_indicator_, 8); lv_obj_add_event_cb(camera_indicator_, camera_preview_toggle_event, LV_EVENT_CLICKED, this); @@ -955,7 +955,8 @@ void DisplayUi::camera_toggle_event(lv_event_t* event) void DisplayUi::camera_preview_toggle_event(lv_event_t* event) { auto* ui = static_cast(lv_event_get_user_data(event)); - if (!ui->camera_preview_ || !ui->camera_preview_available_) return; + if (!ui->camera_preview_ || + !ui->camera_preview_available_.load(std::memory_order_relaxed)) return; const bool enabled = !ui->camera_preview_enabled_.load(std::memory_order_relaxed); ui->camera_preview_enabled_.store(enabled, std::memory_order_relaxed); lv_obj_set_flag(ui->camera_preview_, LV_OBJ_FLAG_HIDDEN, !enabled); @@ -977,10 +978,11 @@ bool DisplayUi::show_camera_state(bool enabled, GestureStatus status, uint8_t pr for (lv_obj_t* part : camera_indicator_parts_) lv_obj_set_style_bg_color(part, color, 0); for (lv_obj_t* slash : camera_indicator_slash_) lv_obj_set_flag(slash, LV_OBJ_FLAG_HIDDEN, !failed); lv_obj_set_flag(gesture_progress_, LV_OBJ_FLAG_HIDDEN, !visible || !progress); - camera_preview_available_ = enabled && visible && status == GestureStatus::Ready; + const bool available = enabled && visible && status == GestureStatus::Ready; + camera_preview_available_.store(available, std::memory_order_relaxed); if (camera_preview_) lv_obj_set_flag(camera_preview_, LV_OBJ_FLAG_HIDDEN, - !camera_preview_available_ || !camera_preview_enabled_.load(std::memory_order_relaxed)); + !available || !camera_preview_enabled_.load(std::memory_order_relaxed)); if (visible && progress) { lv_bar_set_value(gesture_progress_, progress, LV_ANIM_OFF); lv_obj_move_foreground(gesture_progress_); @@ -990,7 +992,7 @@ bool DisplayUi::show_camera_state(bool enabled, GestureStatus status, uint8_t pr void DisplayUi::build_camera_preview(lv_obj_t* screen) { -#if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW +#if CONFIG_MOSAICO_CAMERA_PREVIEW // Half-resolution RGB preview; either rotated dimension is at most 160 pixels. constexpr uint32_t kSide = 160; const uint32_t stride = lv_draw_buf_width_to_stride(kSide, LV_COLOR_FORMAT_RGB565); @@ -1005,9 +1007,6 @@ void DisplayUi::build_camera_preview(lv_obj_t* screen) lv_obj_set_style_border_width(camera_preview_box_, 2, 0); lv_obj_set_style_border_color(camera_preview_box_, lv_color_hex(COLOR_TEXT), 0); lv_obj_add_flag(camera_preview_box_, LV_OBJ_FLAG_HIDDEN); - camera_preview_label_ = make_label(camera_preview_, "", 2, 2, &montserrat_semibold_13, COLOR_TEXT); - lv_obj_set_style_bg_color(camera_preview_label_, lv_color_hex(COLOR_BG), 0); - lv_obj_set_style_bg_opa(camera_preview_label_, LV_OPA_70, 0); lv_obj_add_flag(camera_preview_, LV_OBJ_FLAG_HIDDEN); #else (void)screen; @@ -1017,11 +1016,13 @@ void DisplayUi::build_camera_preview(lv_obj_t* screen) void DisplayUi::show_camera_preview(const uint8_t* rgb, unsigned width, unsigned height, const HandObservation& hand) { if (!camera_preview_enabled_.load(std::memory_order_relaxed) || + !camera_preview_available_.load(std::memory_order_relaxed) || !camera_preview_canvas_ || !rgb || width > 320 || height > 320) return; const uint32_t w = width / 2; const uint32_t h = height / 2; DisplayLock lock; if (!lock) return; + // Visibility can change while this frame waits for the display lock. if (lv_obj_has_flag(camera_preview_, LV_OBJ_FLAG_HIDDEN)) return; const lv_image_dsc_t* image = lv_canvas_get_image(camera_preview_canvas_); if (image->header.w != w || image->header.h != h) { @@ -1045,11 +1046,8 @@ void DisplayUi::show_camera_preview(const uint8_t* rgb, unsigned width, unsigned lv_obj_set_pos(camera_preview_box_, int(hand.x * w) - bw / 2, int(hand.y * h) - bh / 2); lv_obj_set_size(camera_preview_box_, bw, bh); lv_obj_remove_flag(camera_preview_box_, LV_OBJ_FLAG_HIDDEN); - lv_label_set_text_fmt(camera_preview_label_, "N%u S%d OK%d", hand.count, int(hand.score * 100), - int(hand.ok_score * 100)); } else { lv_obj_add_flag(camera_preview_box_, LV_OBJ_FLAG_HIDDEN); - lv_label_set_text(camera_preview_label_, "NO HAND"); } } diff --git a/firmware/targets/esp_mosaico/main/display_ui.hpp b/firmware/targets/esp_mosaico/main/display_ui.hpp index cc8d82d..834430f 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.hpp +++ b/firmware/targets/esp_mosaico/main/display_ui.hpp @@ -166,11 +166,10 @@ class DisplayUi { lv_obj_t* camera_indicator_slash_[2]{}; // Background halo and stroke. lv_obj_t* gesture_progress_ = nullptr; std::atomic camera_preview_enabled_{false}; - bool camera_preview_available_ = false; // Protected by the display lock. + std::atomic camera_preview_available_{false}; // Allows hidden frames to skip the display lock. lv_obj_t* camera_preview_ = nullptr; lv_obj_t* camera_preview_canvas_ = nullptr; lv_obj_t* camera_preview_box_ = nullptr; - lv_obj_t* camera_preview_label_ = nullptr; uint8_t* camera_preview_pixels_ = nullptr; // RGB565 canvas buffer in PSRAM. std::atomic camera_toggle_{false}; uint8_t brightness_percent_ = 50; // Serialized by the LVGL lock. diff --git a/firmware/targets/esp_mosaico/main/gesture_input.cpp b/firmware/targets/esp_mosaico/main/gesture_input.cpp index bb98402..8d0596d 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.cpp @@ -254,7 +254,7 @@ void GestureInput::run() if (context != context_.load()) continue; const GestureAction action = tracker.update(observation, finished_us / 1000); progress_ = tracker.hold_progress(); -#if CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW +#if CONFIG_MOSAICO_CAMERA_PREVIEW if (preview_sink_) preview_sink_(preview_context_, rgb, image.width, image.height, observation); #endif // Calibration aid: normalized observations in percent, at most once per second. diff --git a/firmware/targets/esp_mosaico/main/gesture_input.hpp b/firmware/targets/esp_mosaico/main/gesture_input.hpp index bab2b64..b8a827a 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.hpp @@ -24,7 +24,7 @@ class GestureInput { using PreviewSink = void (*)(void* context, const uint8_t* rgb, unsigned width, unsigned height, const HandObservation& hand); - /** Must be called before begin(); only used with CONFIG_MOSAICO_GESTURE_DEBUG_PREVIEW. */ + /** Must be called before begin(); only used with CONFIG_MOSAICO_CAMERA_PREVIEW. */ void set_preview_sink(PreviewSink sink, void* context) { preview_sink_ = sink; From 6c67c2fec95ec36453264c6d4df134e6b3aff8d6 Mon Sep 17 00:00:00 2001 From: wulu Date: Wed, 30 Sep 2026 10:42:41 +0800 Subject: [PATCH 06/15] fix(mosaico): preserve gesture dependency manifest during migration --- firmware/targets/esp_mosaico/main/idf_component.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/firmware/targets/esp_mosaico/main/idf_component.yml b/firmware/targets/esp_mosaico/main/idf_component.yml index 1e6fc52..2453d44 100644 --- a/firmware/targets/esp_mosaico/main/idf_component.yml +++ b/firmware/targets/esp_mosaico/main/idf_component.yml @@ -1,8 +1,8 @@ dependencies: - # Display support is required by this target. Declare its dependencies here - # because BSP config rules cannot be evaluated before sdkconfig exists. - # Keeping both dependencies in the first pass preserves the locked graph. - espressif/esp_lvgl_adapter: "0.6.2" + # The BSP pulls LVGL only when its display support is enabled, and that rule + # cannot be evaluated on a clean tree, where no sdkconfig exists yet. Naming + # LVGL here resolves it in the Component Manager's first pass so components + # that require it by name are satisfiable from the first configure. lvgl/lvgl: "^9.5.0,<9.6.0" # The display adapter is required before BSP sdkconfig rules can be evaluated. espressif/esp_lvgl_adapter: "0.6.2" From d2b0075a546e92236de15ab548dd0a9c1b9c786c Mon Sep 17 00:00:00 2001 From: wulu Date: Fri, 2 Oct 2026 14:13:02 +0800 Subject: [PATCH 07/15] fix(mosaico): improve continuous gestures and preview timing --- docs/design/mosaico-gesture-input.md | 8 +-- docs/validation/mosaico-gestures.md | 18 ++++- .../esp_mosaico/main/gesture_input.cpp | 28 ++++---- .../esp_mosaico/main/gesture_tracker.cpp | 36 +++++++++- .../esp_mosaico/main/gesture_tracker.hpp | 14 +++- .../test_apps/logic/main/test_gestures.cpp | 72 +++++++++++++------ 6 files changed, 128 insertions(+), 48 deletions(-) diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md index 47e6468..e0ecc61 100644 --- a/docs/design/mosaico-gesture-input.md +++ b/docs/design/mosaico-gesture-input.md @@ -85,14 +85,14 @@ CameraBoard / esp_video 使用时间戳而非固定帧数定义窗口；检测帧率下降不应改变动作语义。bbox 中心先转为归一化屏幕坐标，包含相机安装方向、镜像与当前已应用的屏幕旋转。旋转过程中放弃当前动作。 -状态从 `Idle → Tracking → WaitRelease → Idle`： +挥手状态为 `Idle → Tracking → Cooldown → Idle`；OK 及上下文失效另需 `WaitRelease`： 1. 单个可信目标稳定出现才开始跟踪；以位置连续性和 bbox 尺寸约束保持目标关联。 2. 在限定时间内检查主轴净位移、累计路径方向一致性、垂直于主轴的漂移、置信度与合理 bbox 大小。突然跳变、长时间漏检、模糊的多手关联取消候选。 -3. 完成一次动作只发一个事件，随后等待持续无手且最短冷却已结束再重新接受输入。 +3. 完成一次动作只发一个事件。挥手后经过冷却即可再次挥手，手不必离开；收手窗口内的反向挥动视为收手并忽略。OK、多手及与挥手无关的上下文失效之后，等待持续无手且最短冷却已结束再重新接受输入。 4. 单帧漏检、分类器 `no_hand` 或 worker 暂停不能单独作为“手已离开”的证据。恢复采集后要观察新鲜帧，禁止旧事件回放。 -以下仅为试验起点，不是已验证的产品参数：主轴净位移约对应画面宽度或高度的 25%，动作窗口 150–900 ms，主轴净位移至少为另一轴最大偏移的 2 倍；连续无手约 300 ms，最短冷却约 500 ms；事件有效期约 300 ms。阈值结合有效帧数、视场、距离和漏检率一起校准。持续可见的手不因冷却超时而重复翻页。 +以下仅为试验起点，不是已验证的产品参数：主轴净位移约对应画面宽度或高度的 25%，动作窗口 150–900 ms，主轴净位移至少为另一轴最大偏移的 2 倍；连续无手约 300 ms，最短冷却约 500 ms，收手窗口约 1.5 s，帧间隔上限约 400 ms；事件有效期约 300 ms。阈值结合有效帧数、视场、距离和漏检率一起校准。持续可见的手不因冷却超时而重复翻页。 ## 分阶段工作与完成条件 @@ -159,7 +159,7 @@ eim run "idf.py -C firmware/test_apps/panel_state build" v6.1 上述阶段的复选框保留为端到端验收条件，未勾选不等于没有软件实现。已完成： - 固定 registry 依赖和模型包；左槽 OV3640 采集适配、双缓冲、UYVY 转 RGB888 和四向映射；来源见 [CAMERA_UPSTREAM.md](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)。 -- 单手轨迹、OK 500 ms 停留、离开后再触发、有界事件与上下文失效；主循环实现翻页、只唤醒、幂等进入时钟及受保护状态屏蔽。 +- 单手轨迹、OK 500 ms 停留、挥手冷却与收手抑制、OK 后离开再触发、有界事件与上下文失效；主循环实现翻页、只唤醒、幂等进入时钟及受保护状态屏蔽。 - 设置页独立 `CAMERA` 开关默认关闭，与显示设置共用保存/取消；顶栏摄像头标识区分采集中、暂停与失败，不显示文字提示；停用释放采集与模型，手动重试。ESP-DL 内部部分分配使用断言；资源预检查降低风险，但不保证所有运行时内存不足均能无重启降级。 - 7 MiB 双 OTA 分区、Folder Push 上限和 Bridge 容量配置；无共享模型分区。 - 手势开/关固件、Mosaico logic、共享 panel_state 测试镜像以及 AMOLED 2.16 固件构建通过。已完成手势/屏保主机 Unity 测试的 ASan/UBSan 检查及 Bridge 自动化测试。软件与硬件验证范围见[验证范围](../verification.md)，新增用例的执行情况以[手势验收记录](../validation/mosaico-gestures.md)为准；测试镜像尚未在设备运行。 diff --git a/docs/validation/mosaico-gestures.md b/docs/validation/mosaico-gestures.md index 3594dfe..27870bb 100644 --- a/docs/validation/mosaico-gestures.md +++ b/docs/validation/mosaico-gestures.md @@ -1,6 +1,6 @@ # Mosaico 手势固件：安装与验收 -本功能在设备本地运行：左挥或上挥上一页、右挥或下挥下一页；时钟中首次挥手只恢复原页面；👌 OK 稳定停留约 500 ms 进入时钟。一轮操作最多触发一次，需要手离开后再操作。触摸和 AI 键继续可用。 +本功能在设备本地运行：左挥或上挥上一页、右挥或下挥下一页；时钟中首次挥手只恢复原页面；👌 OK 稳定停留约 500 ms 进入时钟。一次挥动只触发一次；手不必离开画面即可再次挥手，挥手后约 1.5 秒内的反向挥动视为收手而忽略。OK 进入时钟后需要手离开画面再操作。触摸和 AI 键继续可用。 这是供实机验收的实现。软件测试与固件构建不能证明相机成像、方向、识别率、功耗或长期稳定性；这些仍需在目标设备验证。 @@ -83,7 +83,7 @@ SC101IOT 使用完整 1280×720 画面，避免 VGA 模式的中心裁切；等 ## 上下挥手 -上下挥手与左右挥手共用单手置信度、150～900 ms 动作窗口、至少三帧、25% 主轴净位移和释放确认。斜向轨迹及往返抖动不应翻页；时钟中任一方向首次挥动只恢复原页。实机需分别验证上挥上一页、下挥下一页，以及一次动作不会重复翻页；新增四向逻辑尚未实机验收。 +上下挥手与左右挥手共用单手置信度、150～900 ms 动作窗口、至少三帧、25% 主轴净位移和重复触发抑制。斜向轨迹及往返抖动不应翻页；时钟中任一方向首次挥动只恢复原页。实机需分别验证上挥上一页、下挥下一页，以及一次动作不会重复翻页；新增四向逻辑尚未实机验收。 四向手势应用和 logic 测试镜像均通过 ESP-IDF v6.1 构建；扩展后的轨迹、斜向/抖动拒绝及动作路由测试尚未运行。应用已按六段写入设备，全部通过设备哈希校验；仅更新应用分区，未写入 NVS、分区表或 OTA 元数据。烧录结束后设备停留在下载模式，后续日志及用户反馈确认：手离开镜头后，上挥可唤醒时钟；其余方向和连续操作的完整验收仍待完成。 @@ -108,3 +108,17 @@ SC101IOT 使用完整 1280×720 画面，避免 VGA 模式的中心裁切；等 5. 保留至少三帧确认，结合实际帧率评估 150 ms 最短动作时间、位移及位置跳变阈值。分别测量左右与上下挥手的成功率、误触率及唤醒延迟。 近距离优化需区分光学视野与识别条件：当前已使用 SC101IOT 完整画面；软件缩小预览不能扩大视角。可评估较小挥手幅度及大手掌框的过滤阈值，部分手掌入镜的识别可靠性需实测。预览开关的性能对比应采用相同手势、页面与照明条件。 + +## 连续挥手与帧间隔（2026-10-01） + +用户反馈：开启预览时 OK 无法进入时钟；每次动作后要求手离开画面使连续操作迟钝。 + +- 帧间隔上限由 250 ms 调整为 400 ms。原上限接近实际帧间隔，OK 分类与预览使间隔变长后，每一帧都会重置跟踪和 OK 停留计时。该原因由代码和上文帧间隔记录推断，未取得开启预览时的分阶段耗时日志。 +- 动作事件在预览绘制之前入队，预览等待显示锁不再占用事件 300 ms 的有效期。 +- 挥手后不再要求手离开：500 ms 冷却内的样本属于同一次挥动，不重复触发；1.5 秒内的反向挥动视为收手并忽略，同向和垂直方向可再次触发。挥手引起的页面或屏保变化不要求重新确认手离开。 +- OK 进入时钟、多手、触摸、AI 键、连接变化以及设置/配对/OTA 恢复后，仍要求连续约 300 ms 无手。 +- 帧处理期间上下文变化时，该帧不再作为输入，但仍绘制预览；此前这些帧既不识别也不显示。挥手后预览停顿是否由此引起尚未在实机确认，页面重绘对显示锁和 CPU 的占用未测量。 + +已知取舍：挥手后 1.5 秒内有意做的反向挥手会被忽略；收手晚于 1.5 秒会被识别为反向翻页。两个时间均为试验起点，需按实机误触和漏触发调整。 + +手势用例通过临时 Unity 兼容头在主机编译运行，9 条全部通过 ASan/UBSan；新增的 300 ms 帧间隔 OK 用例在 250 ms 上限下失败。应用与 logic 测试镜像（esp32s31）通过 ESP-IDF v6.1 构建。测试镜像未在 MCU 上运行；预览下的 OK、连续挥手、收手抑制和误触率均待实机验收。 diff --git a/firmware/targets/esp_mosaico/main/gesture_input.cpp b/firmware/targets/esp_mosaico/main/gesture_input.cpp index 8d0596d..c07f796 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.cpp @@ -113,7 +113,7 @@ void GestureInput::run() const bool enabled = context & kEnabled; const bool allowed = context & kAllowed; if (context != previous_context) { - tracker.reset(); + tracker.reset(esp_timer_get_time() / 1000); progress_ = 0; previous_context = context; } @@ -190,12 +190,12 @@ void GestureInput::run() continue; } detector->set_score_thr(0.5F); - tracker.reset(); + tracker.reset(esp_timer_get_time() / 1000); active = true; capture_failures = 0; warmup = 3; status_ = GestureStatus::Ready; - ESP_LOGI(TAG, "local gestures ready; remove hand before starting a new action"); + ESP_LOGI(TAG, "local gestures ready; input starts after the view is clear of hands"); } const uint64_t started_us = esp_timer_get_time(); // Display rotation maps logical pixels to physical coordinates; camera @@ -208,7 +208,7 @@ void GestureInput::run() const bool mirror = false; #endif if (!camera.read_rgb(rgb, rotation, mirror)) { - tracker.reset(); + tracker.reset(esp_timer_get_time() / 1000); progress_ = 0; if (++capture_failures >= 3) { record(camera.failure(), camera.failure_code()); @@ -220,7 +220,6 @@ void GestureInput::run() } capture_failures = 0; if (warmup) { --warmup; continue; } - if (context != context_.load()) continue; const auto image_size = camera.image_size(rotation); dl::image::img_t image{rgb, static_cast(image_size.width), @@ -251,10 +250,18 @@ void GestureInput::run() } } const uint64_t finished_us = esp_timer_get_time(); - if (context != context_.load()) continue; - const GestureAction action = tracker.update(observation, finished_us / 1000); - progress_ = tracker.hold_progress(); + // A context change during the frame discards it as input; it is still previewed. + const bool stale = context != context_.load(); + const GestureAction action = stale ? GestureAction::None + : tracker.update(observation, finished_us / 1000); + if (!stale) progress_ = tracker.hold_progress(); + if (action != GestureAction::None) { + ESP_LOGI(TAG, "tracker action=%u", unsigned(action)); + const GestureEvent event{action, context, finished_us / 1000}; + xQueueSend(queue_, &event, 0); + } #if CONFIG_MOSAICO_CAMERA_PREVIEW + // The sink waits for the display lock; events are queued first because they expire. if (preview_sink_) preview_sink_(preview_context_, rgb, image.width, image.height, observation); #endif // Calibration aid: normalized observations in percent, at most once per second. @@ -265,11 +272,6 @@ void GestureInput::run() int(observation.height * 100), int(observation.ok_score * 100)); next_detection_log = finished_us / 1000 + 1000; } - if (action != GestureAction::None) { - ESP_LOGI(TAG, "tracker action=%u", unsigned(action)); - const GestureEvent event{action, context, finished_us / 1000}; - xQueueSend(queue_, &event, 0); - } const uint64_t feedback_done_us = esp_timer_get_time(); capture_us += captured_us - started_us; detect_us += detected_us - captured_us; diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp index b0c0f1e..1ea0bbc 100644 --- a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp @@ -7,7 +7,24 @@ namespace usage_panel::mosaico { namespace { // Dropouts up to this long keep a swipe or OK hold alive. A longer gap restarts // the track from the next usable sample; it does not require the hand to leave. -constexpr uint64_t kMaxSampleGapMs = 250; +// Frames arrive about every 125 ms and slower while the OK classifier or the +// preview runs, so the limit must exceed one frame interval at that cadence. +constexpr uint64_t kMaxSampleGapMs = 400; +// Samples this soon after a swipe belong to the same stroke and cannot repeat it. +constexpr uint64_t kSwipeCooldownMs = 500; +// A swipe opposite to the previous one within this time is the hand returning. +constexpr uint64_t kReturnStrokeMs = 1500; + +GestureAction opposite(GestureAction action) +{ + switch (action) { + case GestureAction::SwipeLeft: return GestureAction::SwipeRight; + case GestureAction::SwipeRight: return GestureAction::SwipeLeft; + case GestureAction::SwipeUp: return GestureAction::SwipeDown; + case GestureAction::SwipeDown: return GestureAction::SwipeUp; + default: return GestureAction::None; + } +} } GestureEffect route_gesture(GestureAction action, bool allowed, bool asleep) @@ -19,11 +36,15 @@ GestureEffect route_gesture(GestureAction action, bool allowed, bool asleep) ? GestureEffect::Next : GestureEffect::Previous; } -void GestureTracker::reset() +void GestureTracker::reset(uint64_t now) { const uint64_t last_action = last_action_; + const GestureAction last_swipe = last_swipe_; *this = GestureTracker{}; last_action_ = last_action; + last_swipe_ = last_swipe; + // A page or screensaver change within the cooldown is the result of that swipe. + waiting_release_ = last_swipe == GestureAction::None || now - last_action > kSwipeCooldownMs; } GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) @@ -57,6 +78,8 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) absent_ = false; progress_ = 0; if (waiting_release_) return GestureAction::None; + if (last_swipe_ != GestureAction::None && now - last_action_ < kSwipeCooldownMs) + return GestureAction::None; if (tracking_ && (now < last_sample_ || now - last_sample_ > kMaxSampleGapMs || std::fabs(h.x - previous_.x) > std::max(0.18F, (h.width + previous_.width) * 0.75F) || @@ -92,6 +115,11 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) action = dy < 0 ? GestureAction::SwipeUp : GestureAction::SwipeDown; } } + if (action != GestureAction::None && action == opposite(last_swipe_) && + now - last_action_ < kReturnStrokeMs) { + tracking_ = holding_ok_ = false; + return GestureAction::None; + } if (std::isfinite(h.ok_score) && h.ok_score >= 0.85F) { if (!holding_ok_ || std::fabs(h.x - ok_x_) > 0.08F || std::fabs(h.y - ok_y_) > 0.08F) { @@ -106,7 +134,9 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) holding_ok_ = false; } if (action != GestureAction::None) { - waiting_release_ = true; + // Lowering the hand after OK must not wake the clock, so only OK waits for release. + waiting_release_ = action == GestureAction::EnterClock; + last_swipe_ = waiting_release_ ? GestureAction::None : action; last_action_ = now; tracking_ = holding_ok_ = false; progress_ = 0; diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp index ddfb21c..b0f42f3 100644 --- a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp @@ -21,12 +21,19 @@ struct HandObservation { float ok_score = 0; }; -/** Time-based single-hand recognition. A completed action requires release before reuse. */ +/** + * Time-based single-hand recognition. A swipe may be followed by another without + * the hand leaving; the opposite direction is ignored briefly as the hand returning. + * EnterClock and several visible hands require sustained absence of hands before reuse. + */ class GestureTracker { public: GestureAction update(const HandObservation& hand, uint64_t now_ms); - /** Invalidates prior observations and requires fresh, sustained absence of hands. */ - void reset(); + /** + * Invalidates prior observations and requires fresh, sustained absence of hands, + * unless a swipe was just issued: the context change it causes keeps swipes armed. + */ + void reset(uint64_t now_ms); uint8_t hold_progress() const { return progress_; } private: @@ -36,6 +43,7 @@ class GestureTracker { bool holding_ok_ = false; uint64_t absent_since_ = 0; uint64_t last_action_ = 0; + GestureAction last_swipe_ = GestureAction::None; // None after EnterClock. uint64_t started_ = 0; uint64_t last_sample_ = 0; uint64_t ok_since_ = 0; diff --git a/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp b/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp index 903d814..f4735be 100644 --- a/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp +++ b/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp @@ -21,32 +21,49 @@ void arm(GestureTracker& tracker) } } -TEST_CASE("swipes require release and issue exactly one directional command", "[gesture]") +TEST_CASE("swipes repeat without release and ignore the same stroke and the return stroke", "[gesture]") { for (bool vertical : {false, true}) { for (int direction : {-1, 1}) { - const auto point = [vertical](float position) { - return vertical ? hand(0.5F, position) : hand(position); + // Positions are given for the positive direction and mirrored for the negative one. + const auto point = [vertical, direction](float position) { + const float mapped = 0.5F + direction * (position - 0.5F); + return vertical ? hand(0.5F, mapped) : hand(mapped); }; - GestureTracker tracker; - arm(tracker); - TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(point(0.5F), 1500))); - TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(point(0.5F + direction * 0.15F), 1600))); const auto expected = vertical ? (direction < 0 ? GestureAction::SwipeUp : GestureAction::SwipeDown) : (direction < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight); - TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(point(0.5F + direction * 0.3F), 1700))); - for (unsigned t = 1800; t < 3000; t += 100) - TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(point(0.5F), t))); - tracker.update({}, 3000); - tracker.update(point(0.5F), 3100); - TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(point(0.8F), 3300))); - tracker.update({}, 3400); - tracker.update({}, 3800); - tracker.update(point(0.3F), 3900); - tracker.update(point(0.45F), 4000); - TEST_ASSERT_EQUAL_INT(int(vertical ? GestureAction::SwipeDown : GestureAction::SwipeRight), - int(tracker.update(point(0.6F), 4100))); + const auto reverse = vertical + ? (direction < 0 ? GestureAction::SwipeDown : GestureAction::SwipeUp) + : (direction < 0 ? GestureAction::SwipeRight : GestureAction::SwipeLeft); + const auto none = int(GestureAction::None); + GestureTracker tracker; + arm(tracker); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.2F), 1500))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.35F), 1600))); + TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(point(0.5F), 1700))); + tracker.reset(1800); // The resulting page change keeps swipes armed. + // The rest of the same stroke falls inside the cooldown. + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.65F), 1800))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.8F), 1900))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.95F), 2000))); + // The hand returns through the view without leaving it. + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.8F), 2300))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.6F), 2400))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.4F), 2500))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.4F), 2600))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.55F), 2700))); + TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(point(0.7F), 2800))); + // A stationary hand issues nothing; the reverse direction works once the return window ends. + for (unsigned t = 2900; t < 4500; t += 100) + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.7F), t))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.55F), 4500))); + TEST_ASSERT_EQUAL_INT(int(reverse), int(tracker.update(point(0.4F), 4600))); + // A context change unrelated to a swipe requires the view to clear again. + tracker.reset(6000); + tracker.update(point(0.2F), 6100); + tracker.update(point(0.35F), 6200); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.5F), 6300))); } } } @@ -59,7 +76,7 @@ TEST_CASE("OK requires a stable hold and release after context invalidation", "[ TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F, 0.5F, 0.95F), t))); TEST_ASSERT_EQUAL_INT(80, tracker.hold_progress()); TEST_ASSERT_EQUAL_INT(int(GestureAction::EnterClock), int(tracker.update(hand(0.5F, 0.5F, 0.95F), 2000))); - tracker.reset(); + tracker.reset(2000); for (unsigned t = 2100; t <= 3000; t += 100) TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F, 0.5F, 0.95F), t))); } @@ -88,14 +105,23 @@ TEST_CASE("single dropouts keep a swipe and an OK hold alive", "[gesture]") TEST_ASSERT_EQUAL_INT(int(GestureAction::EnterClock), int(tracker.update(hand(0.5F, 0.5F, 0.95F), 2000))); } +TEST_CASE("OK hold completes at a 300 ms frame interval", "[gesture]") +{ + GestureTracker tracker; + arm(tracker); + tracker.update(hand(0.5F, 0.5F, 0.95F), 1500); + tracker.update(hand(0.5F, 0.5F, 0.95F), 1800); + TEST_ASSERT_EQUAL_INT(int(GestureAction::EnterClock), int(tracker.update(hand(0.5F, 0.5F, 0.95F), 2100))); +} + TEST_CASE("slow frames and position jumps restart tracking without requiring release", "[gesture]") { GestureTracker tracker; arm(tracker); tracker.update(hand(0.2F), 1500); - tracker.update(hand(0.2F), 1900); // 400 ms gap: restart here. - tracker.update(hand(0.35F), 2000); - TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.5F), 2100))); + tracker.update(hand(0.2F), 2000); // 500 ms gap: restart here. + tracker.update(hand(0.35F), 2100); + TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.5F), 2200))); tracker.update({}, 2300); tracker.update({}, 2700); From c42739d3731f37ed410235ea39a3833760e1bce1 Mon Sep 17 00:00:00 2001 From: wulu Date: Fri, 2 Oct 2026 22:08:14 +0800 Subject: [PATCH 08/15] feat(mosaico): add wave gesture and single-direction paging Every swipe direction and a horizontal greeting wave now advance to the next page; OK or thumbs-up held for about 500 ms enters the clock. - Recognize a wave as three alternating horizontal legs of at least 12% of the frame, for motion too small to be a swipe. - Issue one command per burst of motion: after a swipe or wave the hand must rest for 400 ms or leave the view. This replaces the 500 ms cooldown. - Anchor the stroke origin to a resting hand so the stroke window opens when the hand starts to move. - Accept a wave without first requiring the view to clear, so the clock can be woken by the hand that entered it. - Skip pose classification while the hand moves between frames, and log every observation with a hand for replay through GestureTracker. - Load the classifier at capture start and mark context changes caused by an accepted gesture explicitly, so other input always requires the view to clear again. Host tests: 13 gesture and image cases pass under ASan/UBSan. The wave thresholds are trial values without device data; recognition and false trigger rates are not validated on hardware. --- docs/validation/mosaico-gestures.md | 44 ++++- .../targets/esp_mosaico/main/app_main.cpp | 8 +- .../esp_mosaico/main/gesture_input.cpp | 50 +++--- .../esp_mosaico/main/gesture_input.hpp | 4 +- .../esp_mosaico/main/gesture_tracker.cpp | 163 ++++++++++++------ .../esp_mosaico/main/gesture_tracker.hpp | 35 ++-- .../test_apps/logic/main/test_gestures.cpp | 113 +++++++++--- 7 files changed, 304 insertions(+), 113 deletions(-) diff --git a/docs/validation/mosaico-gestures.md b/docs/validation/mosaico-gestures.md index 27870bb..d7cac8d 100644 --- a/docs/validation/mosaico-gestures.md +++ b/docs/validation/mosaico-gestures.md @@ -1,6 +1,6 @@ # Mosaico 手势固件：安装与验收 -本功能在设备本地运行：左挥或上挥上一页、右挥或下挥下一页；时钟中首次挥手只恢复原页面；👌 OK 稳定停留约 500 ms 进入时钟。一次挥动只触发一次；手不必离开画面即可再次挥手，挥手后约 1.5 秒内的反向挥动视为收手而忽略。OK 进入时钟后需要手离开画面再操作。触摸和 AI 键继续可用。 +本功能在设备本地运行：任意方向挥手或像打招呼那样左右招手切到下一页，末页后回到首页；时钟中首次挥手或招手只恢复原页面；👌 OK 或 👍 点赞稳定停留约 500 ms 进入时钟。一段连续动作只触发一次，手停住约 0.4 秒或离开画面后才接受下一次；挥手后约 1.5 秒内的反向挥动视为收手而忽略。进入时钟后需要手离开画面再操作。触摸和 AI 键继续可用。 这是供实机验收的实现。软件测试与固件构建不能证明相机成像、方向、识别率、功耗或长期稳定性；这些仍需在目标设备验证。 @@ -30,8 +30,8 @@ eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 1. 长按现有显示设置入口，打开 `CAMERA` 开关并点击 `SAVE`。默认关闭；`CANCEL` 不会启用摄像头。 2. 顶栏摄像头标识变为亮色后，先把手移出画面至少约半秒，再开始动作。 -3. 正常页上下左右挥动；上/左为上一页，下/右为下一页，一次只翻一页。屏保中第一次挥动只恢复原页。 -4. 正常页做 👌 OK 并短暂停留；进度条完成才进入时钟。进入后先移开手，下一轮挥手再唤醒。 +3. 正常页朝任意方向挥动或左右招手均为下一页，一段连续动作只翻一页；再翻前先停住或移开手。屏保中第一次挥动只恢复原页。 +4. 正常页做 👌 OK 或 👍 点赞并短暂停留；进度条完成才进入时钟。进入后先移开手，下一轮挥手再唤醒。 5. 设置、配对确认和 OTA 期间不接受手势，标识转暗。关闭 `CAMERA` 开关并保存后释放采集和模型资源，顶栏标识消失。 顶栏标识带斜杠表示采集失败：相机未能启动或连续采集失败、资源预检查未通过，或驱动资源未能安全释放。检查左槽相机后，把 `CAMERA` 关闭并保存、再打开并保存可重试；仍带斜杠时重启设备。具体原因见 UART0 日志。 @@ -122,3 +122,41 @@ SC101IOT 使用完整 1280×720 画面，避免 VGA 模式的中心裁切；等 已知取舍：挥手后 1.5 秒内有意做的反向挥手会被忽略；收手晚于 1.5 秒会被识别为反向翻页。两个时间均为试验起点，需按实机误触和漏触发调整。 手势用例通过临时 Unity 兼容头在主机编译运行，9 条全部通过 ASan/UBSan；新增的 300 ms 帧间隔 OK 用例在 250 ms 上限下失败。应用与 logic 测试镜像（esp32s31）通过 ESP-IDF v6.1 构建。测试镜像未在 MCU 上运行；预览下的 OK、连续挥手、收手抑制和误触率均待实机验收。 + +## 生命周期与输入失效修复（2026-10-02） + +- 分类器在采集启动时加载模型，确保尚未执行分类就进入设置、配对或 OTA 时也能安全释放。单手分类直接使用 `HandGestureCls::run_crop`；锁定版本的懒加载包装器不保证未加载模型时的析构安全。 +- 应用显式标记已接受手势造成的上下文变化，保留该手势的冷却与释放状态。触摸、AI 键、远程操作及其他上下文变化均重新等待手离开，不再通过距上次挥手的时间推断变化来源。 + +10 条手势与图像用例通过临时 Unity 兼容头在主机运行，ASan/UBSan 无错误；包含冷却期间外部输入阻断后续动作、手离开后恢复识别的回归用例。分类器使用锁定依赖的构造和析构源码、替代推理接口，在填充非零内存的主机分配器下连续创建并释放 20 次，ASan/UBSan 无错误。应用及 logic 测试镜像通过 ESP-IDF v6.1 构建。修复应用已按六段写入设备，并写入匹配的引导程序、7 MiB 双槽分区表和初始 OTA 数据，九项写入均通过设备哈希校验。烧录前后 NVS 区域校验值一致，未执行全片擦除或自动复位。设备保留在下载模式；重启后的相机启停、外部输入后重新识别及连续挥手仍需实机确认。 + +## 单向轮换与点赞进入时钟（2026-10-02） + +- 四个方向的挥手均为下一页，按导航项顺序循环；前文“空中挥手的页面方向”和“上下挥手”中的上一页映射不再适用。触摸滑动仍可双向翻页。 +- 1.5 秒内的反向挥动仍视为收手。所有方向效果相同，因此该规则不再丢弃有意的返回操作；返回上一页需再挥两次，每次间隔不少于 500 ms 冷却。 +- 分类结果为 `ok` 或 `like` 时均计入进入时钟的停留计时，阈值仍为 0.85、停留约 500 ms。分类器只返回置信度最高的一类。 + +手势与图像用例通过临时 Unity 兼容头在主机运行，10 条全部通过 ASan/UBSan，其中路由用例已改为四个方向均为下一页。应用通过 ESP-IDF v6.1 构建，生成配置启用手势输入；logic 测试镜像未重新构建。烧录前读取 OTA 数据确认设备从 ota_0 启动；应用已按六段写入 ota_0，全部通过设备哈希校验，未写入 NVS、分区表、引导程序或 OTA 数据，未自动复位。设备保留在下载模式，重启后的行为尚未确认。点赞分类只经过固件构建，没有主机用例覆盖。握杯、握拳等接近点赞的手形是否误入时钟，以及点赞的识别率和距离，均待实机验收。 + +## 招手识别与挥动采样（2026-10-02） + +用户反馈单向轮换固件仍不灵，期望的动作是像打招呼那样左右招手。此前的判定要求单向净位移达到 25%，并要求净位移不小于路径的 75%，来回摆动因此被拒绝。 + +- 新增招手：水平方向连续三段交替移动、每段不少于画面宽度的 12%，识别为一次下一页；幅度达到 25% 的第一段仍按挥手立即触发。 +- 一段连续动作只触发一次：挥手或招手触发后，需手静止约 400 ms 或离开画面才接受下一次，取代原 500 ms 冷却。低置信度的手视为仍在运动。帧间位移不超过 3% 视为静止。 +- 行程起点跟随静止的手，动作窗口从手开始移动时起算。此前起点固定在手首次出现时，停留后再挥动可能跨过 900 ms 窗口而失效。 +- 相邻两帧位移超过 5% 时跳过手形分类，避免挥动期间帧间隔被分类拉长；手形分类因此比手停稳晚一帧开始。分类耗时尚未在设备上测量，帧率收益未验证。 +- 有手时逐帧输出一行观测日志（千分比坐标及送入跟踪器的时间），可用于在主机回放判定；原为每秒一行。 + +以上阈值均为试验起点，没有设备数据支持。已知风险：静止手的检测抖动若超过 3%，触发一次后只有移开手才能再次触发；鼠标或键盘操作中的手在画面内来回移动可能被识别为招手；招手频率接近帧率的一半时采样可能看不到完整摆幅。 + +13 条手势与图像用例通过临时 Unity 兼容头在主机运行，ASan/UBSan 无错误；新增招手、停留后挥动和分类门控用例，并确认停用招手规则或起点跟随后对应用例失败。原“挥手后无需停顿即可同向重复”的用例已按新规则改写。应用通过 ESP-IDF v6.1 构建；logic 测试镜像未重新构建。烧录前读取 OTA 数据确认设备从 ota_0 启动。第一次写入在第一段约 37% 处 USB 断连，应用区留下不完整镜像，原因未确认；重新进入下载模式后从第一段起重写，六段全部通过设备哈希校验。仅写入 ota_0，未写入 NVS、分区表、引导程序或 OTA 数据，未自动复位；设备保留在下载模式，重启后的行为尚未确认。招手识别率、误触率、连续翻页节奏及分类门控对 OK 和点赞的影响均待实机验收。 + +## 时钟中招手唤醒（2026-10-02） + +用户反馈招手固件在时钟页面很难唤醒。代码上，时钟与普通页面的唯一判定差异是释放要求：OK 或点赞进入时钟后，以及空闲进入时钟引起的上下文变化后，需要连续约 300 ms 检测不到任何手才接受动作；手留在画面内、或有其他目标被检出为手时，挥手和招手都不会被处理。时钟中不显示预览和手部反馈，用户无法判断是否已释放。该原因由代码推断，没有设备日志确认。 + +- 招手不再要求先释放：做完 OK 或点赞的手不离开画面，直接招手即可唤醒；触摸、按键等外部输入之后的招手同样有效。 +- 单向挥手和 OK、点赞仍要求先释放，避免做完手形后放下手被当作挥手而立刻唤醒。 + +13 条用例在主机通过 ASan/UBSan，OK 用例增加了“手未离开时挥手无效、招手有效”的断言。应用通过 ESP-IDF v6.1 构建。烧录前读取 OTA 数据确认设备从 ota_0 启动；应用按六段写入 ota_0，全部通过设备哈希校验，写入过程无断连，未写入 NVS、分区表、引导程序或 OTA 数据，未请求复位。时钟唤醒率及放下手时的误唤醒待实机验收。 diff --git a/firmware/targets/esp_mosaico/main/app_main.cpp b/firmware/targets/esp_mosaico/main/app_main.cpp index 4dde6fd..c6be0fd 100644 --- a/firmware/targets/esp_mosaico/main/app_main.cpp +++ b/firmware/targets/esp_mosaico/main/app_main.cpp @@ -503,15 +503,15 @@ extern "C" void app_main(void) if (!screensaver.view(now_ms).active) page = screensaver.restore_page(); } else if (effect == GestureEffect::Clock) { screensaver.enter_clock(now_ms, page); - } else if (effect == GestureEffect::Next || effect == GestureEffect::Previous) { - page = page_after_swipe(page, effect == GestureEffect::Next - ? SwipeDirection::Next : SwipeDirection::Previous); + } else if (effect == GestureEffect::Next) { + page = page_after_swipe(page, SwipeDirection::Next); screensaver.reset(now_ms, page); } if (effect != GestureEffect::None) { ui.request_page(page); status_page.store(page, std::memory_order_relaxed); - gestures.invalidate(); + gestures.configure(camera_enabled, gesture_allowed, + screensaver.view(now_ms).active, page, applied_rotation, true); ESP_LOGI(TAG, "gesture action=%u effect=%u", unsigned(gesture.action), unsigned(effect)); dirty = true; } diff --git a/firmware/targets/esp_mosaico/main/gesture_input.cpp b/firmware/targets/esp_mosaico/main/gesture_input.cpp index c07f796..2dff98d 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.cpp @@ -6,6 +6,7 @@ #include "freertos/task.h" #if CONFIG_MOSAICO_GESTURE_INPUT +#include #include #include #include @@ -25,7 +26,8 @@ constexpr uint32_t kEnabled = 1; constexpr uint32_t kAllowed = 2; constexpr uint32_t kAsleep = 4; constexpr uint32_t kContextMask = 0x7f; -constexpr uint32_t kGeneration = 0x80; +constexpr uint32_t kAcceptedGesture = 0x80; +constexpr uint32_t kGeneration = 0x100; constexpr char TAG[] = "gesture_input"; // Transient start-up failures are retried as in esp-vision's camera init. constexpr unsigned kCameraStartAttempts = 3; @@ -47,20 +49,22 @@ bool GestureInput::begin() return true; } -void GestureInput::configure(bool enabled, bool allowed, bool asleep, Page page, ScreenRotation rotation) +void GestureInput::configure(bool enabled, bool allowed, bool asleep, Page page, ScreenRotation rotation, + bool accepted_gesture) { const uint32_t flags = (enabled ? kEnabled : 0) | (allowed ? kAllowed : 0) | (asleep ? kAsleep : 0) | (uint32_t(rotation) << 3) | (uint32_t(page) << 5); const uint32_t old = context_.load(); - if ((old & kContextMask) != flags) { - context_ = ((old + kGeneration) & ~kContextMask) | flags; + if ((old & kContextMask) != flags || accepted_gesture) { + context_ = ((old + kGeneration) & ~(kContextMask | kAcceptedGesture)) | flags | + (accepted_gesture ? kAcceptedGesture : 0); progress_ = 0; } } void GestureInput::invalidate() { - context_.fetch_add(kGeneration); + context_ = (context_.load() + kGeneration) & ~kAcceptedGesture; progress_ = 0; } @@ -84,7 +88,7 @@ void GestureInput::run() GestureCamera camera; GestureTracker tracker; std::unique_ptr detector; - std::unique_ptr recognizer; + std::unique_ptr recognizer; uint8_t* rgb = nullptr; uint32_t previous_context = UINT32_MAX; bool active = false; @@ -92,7 +96,7 @@ void GestureInput::run() bool fatal_cleanup = false; unsigned capture_failures = 0; uint64_t next_report = 0; - uint64_t next_detection_log = 0; + HandObservation previous_observation{}; unsigned measured_frames = 0; uint64_t measured_us = 0; uint64_t capture_us = 0, detect_us = 0, classify_us = 0, feedback_us = 0; @@ -113,7 +117,7 @@ void GestureInput::run() const bool enabled = context & kEnabled; const bool allowed = context & kAllowed; if (context != previous_context) { - tracker.reset(esp_timer_get_time() / 1000); + tracker.reset((context & kAcceptedGesture) != 0); progress_ = 0; previous_context = context; } @@ -182,7 +186,8 @@ void GestureInput::run() continue; } detector.reset(new (std::nothrow) HandDetect(HandDetect::ESPDET_PICO_224_224_HAND, false)); - recognizer.reset(new (std::nothrow) HandGestureRecognizer()); + // ESP-DL 3.3.11 requires the classifier model to be loaded before destruction. + recognizer.reset(new (std::nothrow) HandGestureCls(HandGestureCls::MOBILENETV2_0_5_S8_V1, false)); if (!detector || !recognizer) { record("model allocation", 0); status_ = GestureStatus::MemoryError; @@ -190,7 +195,7 @@ void GestureInput::run() continue; } detector->set_score_thr(0.5F); - tracker.reset(esp_timer_get_time() / 1000); + tracker.reset(); active = true; capture_failures = 0; warmup = 3; @@ -208,7 +213,7 @@ void GestureInput::run() const bool mirror = false; #endif if (!camera.read_rgb(rgb, rotation, mirror)) { - tracker.reset(esp_timer_get_time() / 1000); + tracker.reset(); progress_ = 0; if (++capture_failures >= 3) { record(camera.failure(), camera.failure_code()); @@ -239,16 +244,19 @@ void GestureInput::run() observation.height = float(hand.box[3] - hand.box[1]) / image.height; observation.score = hand.score; if (!(context & kAsleep) && hand.score >= 0.65F && + hand_is_steady(previous_observation, observation) && hand.box[0] >= 0 && hand.box[1] >= 0 && hand.box[2] <= image.width && hand.box[3] <= image.height && hand.box[2] > hand.box[0] && hand.box[3] > hand.box[1]) { - const auto classes = recognizer->recognize(image, hands); + const auto classes = recognizer->run_crop(image, hand.box); for (const auto& result : classes) { - if (result.cat_name && std::strcmp(result.cat_name, "ok") == 0) - observation.ok_score = result.score; + if (result.cat_name && (std::strcmp(result.cat_name, "ok") == 0 || + std::strcmp(result.cat_name, "like") == 0)) + observation.ok_score = std::max(observation.ok_score, result.score); } } } } + previous_observation = observation; const uint64_t finished_us = esp_timer_get_time(); // A context change during the frame discards it as input; it is still previewed. const bool stale = context != context_.load(); @@ -264,13 +272,13 @@ void GestureInput::run() // The sink waits for the display lock; events are queued first because they expire. if (preview_sink_) preview_sink_(preview_context_, rgb, image.width, image.height, observation); #endif - // Calibration aid: normalized observations in percent, at most once per second. - if (observation.count && finished_us / 1000 >= next_detection_log) { - ESP_LOGI(TAG, "hand count=%u score=%d center=(%d,%d) size=(%d,%d) ok=%d", - observation.count, int(observation.score * 100), int(observation.x * 100), - int(observation.y * 100), int(observation.width * 100), - int(observation.height * 100), int(observation.ok_score * 100)); - next_detection_log = finished_us / 1000 + 1000; + // Calibration aid: every observation with a hand, in per mille with the time given to + // the tracker, so that a captured log can be replayed through GestureTracker. + if (observation.count) { + ESP_LOGI(TAG, "hand t=%u n=%u s=%d c=%d,%d wh=%d,%d ok=%d", + unsigned(finished_us / 1000), observation.count, int(observation.score * 1000), + int(observation.x * 1000), int(observation.y * 1000), int(observation.width * 1000), + int(observation.height * 1000), int(observation.ok_score * 1000)); } const uint64_t feedback_done_us = esp_timer_get_time(); capture_us += captured_us - started_us; diff --git a/firmware/targets/esp_mosaico/main/gesture_input.hpp b/firmware/targets/esp_mosaico/main/gesture_input.hpp index b8a827a..624bc72 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.hpp @@ -31,7 +31,9 @@ class GestureInput { preview_context_ = context; } bool begin(); - void configure(bool enabled, bool allowed, bool asleep, Page page, ScreenRotation rotation); + /** Main task only. accepted_gesture marks a context change caused by a consumed gesture. */ + void configure(bool enabled, bool allowed, bool asleep, Page page, ScreenRotation rotation, + bool accepted_gesture = false); /** Cancels pending recognition after another input is accepted. Main task only. */ void invalidate(); bool take(GestureEvent& event, uint64_t now_ms); diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp index 1ea0bbc..d2d6e2d 100644 --- a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp @@ -10,10 +10,21 @@ namespace { // Frames arrive about every 125 ms and slower while the OK classifier or the // preview runs, so the limit must exceed one frame interval at that cadence. constexpr uint64_t kMaxSampleGapMs = 400; -// Samples this soon after a swipe belong to the same stroke and cannot repeat it. -constexpr uint64_t kSwipeCooldownMs = 500; +// After a swipe or wave, motion must cease for this long before the next command, so the +// follow-through, an immediate return and the rest of a longer wave cannot repeat it. +constexpr uint64_t kSettleMs = 400; // A swipe opposite to the previous one within this time is the hand returning. constexpr uint64_t kReturnStrokeMs = 1500; +constexpr uint64_t kOkHoldMs = 500; +// Samples of one hold are at most kMaxSampleGapMs apart, so a completed hold spans three. +static_assert(kMaxSampleGapMs < kOkHoldMs); +// Frame-to-frame movement up to this distance is detector jitter around a resting hand. +constexpr float kRestStep = 0.03F; +// Beyond this frame-to-frame movement the hand is mid-stroke and its pose is not classified. +constexpr float kSteadyStep = 0.05F; +// A wave is three alternating horizontal legs of at least this length. Sampling at about +// 8 fps sees roughly two thirds of the true travel between reversals. +constexpr float kWaveLeg = 0.12F; GestureAction opposite(GestureAction action) { @@ -32,19 +43,41 @@ GestureEffect route_gesture(GestureAction action, bool allowed, bool asleep) if (!allowed || action == GestureAction::None) return GestureEffect::None; if (action == GestureAction::EnterClock) return asleep ? GestureEffect::None : GestureEffect::Clock; if (asleep) return GestureEffect::Wake; - return action == GestureAction::SwipeRight || action == GestureAction::SwipeDown - ? GestureEffect::Next : GestureEffect::Previous; + // Every direction advances, so the tracker's return-stroke window never drops a command. + return GestureEffect::Next; } -void GestureTracker::reset(uint64_t now) +bool hand_is_steady(const HandObservation& previous, const HandObservation& current) +{ + return previous.count == 1 && current.count == 1 && + std::fabs(current.x - previous.x) <= kSteadyStep && + std::fabs(current.y - previous.y) <= kSteadyStep; +} + +void GestureTracker::reset(bool accepted_gesture) { const uint64_t last_action = last_action_; + const uint64_t last_motion = last_motion_; const GestureAction last_swipe = last_swipe_; + const bool waiting_release = waiting_release_; + const bool settling = settling_; *this = GestureTracker{}; - last_action_ = last_action; - last_swipe_ = last_swipe; - // A page or screensaver change within the cooldown is the result of that swipe. - waiting_release_ = last_swipe == GestureAction::None || now - last_action > kSwipeCooldownMs; + if (accepted_gesture) { + last_action_ = last_action; + last_motion_ = last_motion; + last_swipe_ = last_swipe; + waiting_release_ = waiting_release; + settling_ = settling; + } +} + +void GestureTracker::begin_stroke(const HandObservation& h, uint64_t now) +{ + started_ = now; + samples_ = 1; + start_x_ = h.x; + start_y_ = h.y; + path_x_ = path_y_ = max_dx_ = max_dy_ = 0; } GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) @@ -68,8 +101,10 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) if (now - absent_since_ >= 300 && now - last_action_ >= 500) waiting_release_ = false; } else { - // A weakly detected hand is still present and cannot count as a release. + // A weakly detected hand is still present and cannot count as a release. Motion + // blur lowers the score, so it does not count as rest either. absent_ = false; + last_motion_ = now; } if (tracking_ && now - last_sample_ > kMaxSampleGapMs) tracking_ = holding_ok_ = false; if (!holding_ok_) progress_ = 0; @@ -77,59 +112,80 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) } absent_ = false; progress_ = 0; - if (waiting_release_) return GestureAction::None; - if (last_swipe_ != GestureAction::None && now - last_action_ < kSwipeCooldownMs) - return GestureAction::None; - if (tracking_ && (now < last_sample_ || now - last_sample_ > kMaxSampleGapMs || - std::fabs(h.x - previous_.x) > std::max(0.18F, (h.width + previous_.width) * 0.75F) || - std::fabs(h.y - previous_.y) > std::max(0.18F, (h.height + previous_.height) * 0.75F))) { - tracking_ = holding_ok_ = false; + const bool continuous = tracking_ && now >= last_sample_ && now - last_sample_ <= kMaxSampleGapMs && + std::fabs(h.x - previous_.x) <= std::max(0.18F, (h.width + previous_.width) * 0.75F) && + std::fabs(h.y - previous_.y) <= std::max(0.18F, (h.height + previous_.height) * 0.75F); + const float step_x = continuous ? h.x - previous_.x : 0; + const float step_y = continuous ? h.y - previous_.y : 0; + tracking_ = true; + previous_ = h; + last_sample_ = now; + if (std::fabs(step_x) > kRestStep || std::fabs(step_y) > kRestStep) last_motion_ = now; + const bool rested = now - last_motion_ >= kSettleMs; + + if (!continuous) holding_ok_ = false; + if (!continuous || rested) { + leg_x_ = 0; + legs_ = 0; } - if (!tracking_) { - tracking_ = true; - started_ = now; - samples_ = 0; - start_x_ = h.x; - start_y_ = h.y; - path_x_ = path_y_ = max_dx_ = max_dy_ = 0; - previous_ = h; + if (settling_ && rested) settling_ = false; + if (!continuous || rested) { + begin_stroke(h, now); + } else { + ++samples_; + path_x_ += std::fabs(step_x); + path_y_ += std::fabs(step_y); + max_dx_ = std::max(max_dx_, std::fabs(h.x - start_x_)); + max_dy_ = std::max(max_dy_, std::fabs(h.y - start_y_)); + // A hand still at its origin has not started a stroke. The origin follows it, so the + // stroke window opens when the hand starts to move rather than when it appeared. + if (max_dx_ <= kRestStep && max_dy_ <= kRestStep) begin_stroke(h, now); + } + if (std::fabs(step_x) > kRestStep) { + if (leg_x_ * step_x < 0) { + if (std::fabs(leg_x_) >= kWaveLeg) ++legs_; + leg_x_ = 0; + } + leg_x_ += step_x; } - ++samples_; - path_x_ += std::fabs(h.x - previous_.x); - path_y_ += std::fabs(h.y - previous_.y); - max_dx_ = std::max(max_dx_, std::fabs(h.x - start_x_)); - max_dy_ = std::max(max_dy_, std::fabs(h.y - start_y_)); const float dx = h.x - start_x_; const float dy = h.y - start_y_; - previous_ = h; - last_sample_ = now; GestureAction action = GestureAction::None; - if (samples_ >= 3 && now - started_ >= 150 && now - started_ <= 900) { - if (std::fabs(dx) >= 0.25F && std::fabs(dx) >= 2 * max_dy_ && - max_dy_ <= 0.15F && std::fabs(dx) >= path_x_ * 0.75F) { - action = dx < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight; - } else if (std::fabs(dy) >= 0.25F && std::fabs(dy) >= 2 * max_dx_ && - max_dx_ <= 0.15F && std::fabs(dy) >= path_y_ * 0.75F) { - action = dy < 0 ? GestureAction::SwipeUp : GestureAction::SwipeDown; + if (!settling_) { + // Until the hand has left the view, a stroke may be the hand withdrawing after a pose + // or a touch. A wave cannot be, so it alone is accepted without that release. + if (!waiting_release_ && samples_ >= 3 && now - started_ >= 150 && now - started_ <= 900) { + if (std::fabs(dx) >= 0.25F && std::fabs(dx) >= 2 * max_dy_ && + max_dy_ <= 0.15F && std::fabs(dx) >= path_x_ * 0.75F) { + action = dx < 0 ? GestureAction::SwipeLeft : GestureAction::SwipeRight; + } else if (std::fabs(dy) >= 0.25F && std::fabs(dy) >= 2 * max_dx_ && + max_dx_ <= 0.15F && std::fabs(dy) >= path_y_ * 0.75F) { + action = dy < 0 ? GestureAction::SwipeUp : GestureAction::SwipeDown; + } + } + if (action != GestureAction::None && action == opposite(last_swipe_) && + now - last_action_ < kReturnStrokeMs) { + begin_stroke(h, now); + holding_ok_ = false; + return GestureAction::None; + } + if (action == GestureAction::None && legs_ >= 2 && std::fabs(leg_x_) >= kWaveLeg && + path_x_ >= 2 * path_y_) { + action = GestureAction::Wave; } - } - if (action != GestureAction::None && action == opposite(last_swipe_) && - now - last_action_ < kReturnStrokeMs) { - tracking_ = holding_ok_ = false; - return GestureAction::None; } - if (std::isfinite(h.ok_score) && h.ok_score >= 0.85F) { + if (!waiting_release_ && std::isfinite(h.ok_score) && h.ok_score >= 0.85F) { if (!holding_ok_ || std::fabs(h.x - ok_x_) > 0.08F || std::fabs(h.y - ok_y_) > 0.08F) { holding_ok_ = true; ok_since_ = now; ok_x_ = h.x; ok_y_ = h.y; } - progress_ = static_cast(std::min(100, (now - ok_since_) / 5)); - if (now - ok_since_ >= 500 && samples_ >= 3) action = GestureAction::EnterClock; + progress_ = static_cast(std::min(100, (now - ok_since_) * 100 / kOkHoldMs)); + if (now - ok_since_ >= kOkHoldMs) action = GestureAction::EnterClock; } else { holding_ok_ = false; } @@ -137,12 +193,15 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) // Lowering the hand after OK must not wake the clock, so only OK waits for release. waiting_release_ = action == GestureAction::EnterClock; last_swipe_ = waiting_release_ ? GestureAction::None : action; - last_action_ = now; - tracking_ = holding_ok_ = false; + last_action_ = last_motion_ = now; + settling_ = !waiting_release_; + holding_ok_ = false; progress_ = 0; - } else if (now - started_ > 900 && !holding_ok_) { - // A stationary hand may begin a later deliberate swipe without disappearing. - tracking_ = false; + leg_x_ = 0; + legs_ = 0; + } else if (now - started_ > 900) { + // Motion that qualified as nothing so far must not hold back a later stroke. + begin_stroke(h, now); } return action; } diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp index b0f42f3..70af87a 100644 --- a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp @@ -4,8 +4,8 @@ namespace usage_panel::mosaico { -enum class GestureAction : uint8_t { None, SwipeLeft, SwipeRight, EnterClock, SwipeUp, SwipeDown }; -enum class GestureEffect : uint8_t { None, Wake, Previous, Next, Clock }; +enum class GestureAction : uint8_t { None, SwipeLeft, SwipeRight, EnterClock, SwipeUp, SwipeDown, Wave }; +enum class GestureEffect : uint8_t { None, Wake, Next, Clock }; /** Applies application input policy before any page or screensaver mutation. */ GestureEffect route_gesture(GestureAction action, bool allowed, bool asleep); @@ -18,31 +18,44 @@ struct HandObservation { float width = 0; float height = 0; float score = 0; - float ok_score = 0; + float ok_score = 0; // Confidence of a clock pose: OK or thumbs-up. }; /** - * Time-based single-hand recognition. A swipe may be followed by another without - * the hand leaving; the opposite direction is ignored briefly as the hand returning. - * EnterClock and several visible hands require sustained absence of hands before reuse. + * True when one hand moved little enough between consecutive frames for its pose to be + * classified. Classifying a moving hand lengthens the frame interval and costs a stroke samples. + */ +bool hand_is_steady(const HandObservation& previous, const HandObservation& current); + +/** + * Time-based single-hand recognition. A straight stroke is a swipe; a horizontal + * back-and-forth motion too small for a swipe is a Wave. One burst of motion issues one + * command: the next requires the hand to rest or leave the view, and a stroke opposite to + * the previous swipe is ignored briefly as the hand returning. + * After EnterClock, several visible hands or reset(), swipes and poses require sustained + * absence of hands first; a wave does not. */ class GestureTracker { public: GestureAction update(const HandObservation& hand, uint64_t now_ms); /** - * Invalidates prior observations and requires fresh, sustained absence of hands, - * unless a swipe was just issued: the context change it causes keeps swipes armed. + * Invalidates prior observations; swipes and poses then require sustained absence of hands. + * An accepted gesture's context change may preserve its settling and release state. */ - void reset(uint64_t now_ms); + void reset(bool accepted_gesture = false); uint8_t hold_progress() const { return progress_; } private: + void begin_stroke(const HandObservation& hand, uint64_t now_ms); + bool waiting_release_ = true; bool absent_ = false; - bool tracking_ = false; + bool tracking_ = false; // previous_ and last_sample_ describe the same hand. bool holding_ok_ = false; + bool settling_ = false; // A swipe or wave was issued and its motion has not ceased. uint64_t absent_since_ = 0; uint64_t last_action_ = 0; + uint64_t last_motion_ = 0; GestureAction last_swipe_ = GestureAction::None; // None after EnterClock. uint64_t started_ = 0; uint64_t last_sample_ = 0; @@ -56,6 +69,8 @@ class GestureTracker { float path_y_ = 0; float max_dx_ = 0; float max_dy_ = 0; + float leg_x_ = 0; // Signed horizontal travel since the last reversal. + unsigned legs_ = 0; // Completed horizontal legs long enough to belong to a wave. float ok_x_ = 0; float ok_y_ = 0; }; diff --git a/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp b/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp index f4735be..be5efe3 100644 --- a/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp +++ b/firmware/targets/esp_mosaico/test_apps/logic/main/test_gestures.cpp @@ -21,7 +21,7 @@ void arm(GestureTracker& tracker) } } -TEST_CASE("swipes repeat without release and ignore the same stroke and the return stroke", "[gesture]") +TEST_CASE("one burst of motion is one swipe and the return stroke is ignored", "[gesture]") { for (bool vertical : {false, true}) { for (int direction : {-1, 1}) { @@ -42,25 +42,29 @@ TEST_CASE("swipes repeat without release and ignore the same stroke and the retu TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.2F), 1500))); TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.35F), 1600))); TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(point(0.5F), 1700))); - tracker.reset(1800); // The resulting page change keeps swipes armed. - // The rest of the same stroke falls inside the cooldown. - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.65F), 1800))); - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.8F), 1900))); - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.95F), 2000))); - // The hand returns through the view without leaving it. - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.8F), 2300))); - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.6F), 2400))); - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.4F), 2500))); - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.4F), 2600))); - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.55F), 2700))); - TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(point(0.7F), 2800))); + tracker.reset(true); // The accepted swipe's page change keeps swipes armed. + // The hand comes to rest in view; the stroke back is the hand returning. + for (unsigned t = 1800; t < 2400; t += 100) + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.65F), t))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.5F), 2400))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.35F), 2500))); + // The same direction repeats without the hand leaving. + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.5F), 2600))); + TEST_ASSERT_EQUAL_INT(int(expected), int(tracker.update(point(0.65F), 2700))); + tracker.reset(true); + // Motion that continues without a rest belongs to the same command. + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.8F), 2800))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.5F), 2900))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.2F), 3000))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.5F), 3100))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.8F), 3200))); // A stationary hand issues nothing; the reverse direction works once the return window ends. - for (unsigned t = 2900; t < 4500; t += 100) - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.7F), t))); - TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.55F), 4500))); - TEST_ASSERT_EQUAL_INT(int(reverse), int(tracker.update(point(0.4F), 4600))); + for (unsigned t = 3300; t < 4500; t += 100) + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.8F), t))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.65F), 4500))); + TEST_ASSERT_EQUAL_INT(int(reverse), int(tracker.update(point(0.5F), 4600))); // A context change unrelated to a swipe requires the view to clear again. - tracker.reset(6000); + tracker.reset(); tracker.update(point(0.2F), 6100); tracker.update(point(0.35F), 6200); TEST_ASSERT_EQUAL_INT(none, int(tracker.update(point(0.5F), 6300))); @@ -68,6 +72,44 @@ TEST_CASE("swipes repeat without release and ignore the same stroke and the retu } } +TEST_CASE("a wave too small for a swipe issues one command until the hand rests", "[gesture]") +{ + const auto none = int(GestureAction::None); + GestureTracker tracker; + arm(tracker); + // Frames arrive about every 125 ms; each leg covers 15% of the width. + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(hand(0.42F), 1500))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(hand(0.57F), 1625))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(hand(0.42F), 1750))); + TEST_ASSERT_EQUAL_INT(int(GestureAction::Wave), int(tracker.update(hand(0.57F), 1875))); + tracker.reset(true); + for (unsigned i = 0; i < 6; ++i) + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(hand(i % 2 ? 0.57F : 0.42F), 2000 + i * 125))); + for (unsigned t = 2750; t <= 3250; t += 125) + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(hand(0.57F), t))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(hand(0.42F), 3375))); + TEST_ASSERT_EQUAL_INT(none, int(tracker.update(hand(0.57F), 3500))); + TEST_ASSERT_EQUAL_INT(int(GestureAction::Wave), int(tracker.update(hand(0.42F), 3625))); +} + +TEST_CASE("a resting hand starts its stroke when it begins to move", "[gesture]") +{ + GestureTracker tracker; + arm(tracker); + for (unsigned t = 1500; t <= 2300; t += 100) + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.2F), t))); + tracker.update(hand(0.35F), 2400); + TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.5F), 2500))); +} + +TEST_CASE("pose classification waits for a hand that stays in place", "[gesture]") +{ + TEST_ASSERT_TRUE(hand_is_steady(hand(0.5F), hand(0.53F, 0.52F))); + TEST_ASSERT_FALSE(hand_is_steady(hand(0.5F), hand(0.6F))); + TEST_ASSERT_FALSE(hand_is_steady(hand(0.5F), hand(0.5F, 0.6F))); + TEST_ASSERT_FALSE(hand_is_steady({}, hand(0.5F))); +} + TEST_CASE("OK requires a stable hold and release after context invalidation", "[gesture]") { GestureTracker tracker; @@ -76,9 +118,34 @@ TEST_CASE("OK requires a stable hold and release after context invalidation", "[ TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F, 0.5F, 0.95F), t))); TEST_ASSERT_EQUAL_INT(80, tracker.hold_progress()); TEST_ASSERT_EQUAL_INT(int(GestureAction::EnterClock), int(tracker.update(hand(0.5F, 0.5F, 0.95F), 2000))); - tracker.reset(2000); + tracker.reset(true); for (unsigned t = 2100; t <= 3000; t += 100) TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.5F, 0.5F, 0.95F), t))); + // The hand that made the pose never left: its stroke is ignored, its wave is not. + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.65F), 3100))); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.8F), 3200))); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.65F), 3300))); + TEST_ASSERT_EQUAL_INT(int(GestureAction::Wave), int(tracker.update(hand(0.8F), 3400))); +} + +TEST_CASE("external input during swipe cooldown requires release before another gesture", "[gesture]") +{ + GestureTracker tracker; + arm(tracker); + tracker.update(hand(0.2F), 1500); + tracker.update(hand(0.35F), 1600); + TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.5F), 1700))); + tracker.reset(true); + tracker.update(hand(0.65F), 1800); + tracker.reset(); + tracker.update(hand(0.5F), 2200); + tracker.update(hand(0.65F), 2300); + TEST_ASSERT_EQUAL_INT(int(GestureAction::None), int(tracker.update(hand(0.8F), 2400))); + tracker.update({}, 2500); + tracker.update({}, 2800); + tracker.update(hand(0.2F), 2900); + tracker.update(hand(0.35F), 3000); + TEST_ASSERT_EQUAL_INT(int(GestureAction::SwipeRight), int(tracker.update(hand(0.5F), 3100))); } TEST_CASE("single dropouts keep a swipe and an OK hold alive", "[gesture]") @@ -154,16 +221,18 @@ TEST_CASE("wake consumes the first swipe and protected states reject all command { TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeLeft, true, true))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeRight, true, true))); - TEST_ASSERT_EQUAL_INT(int(GestureEffect::Previous), int(route_gesture(GestureAction::SwipeLeft, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::SwipeLeft, true, false))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::SwipeRight, true, false))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeUp, true, true))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::SwipeDown, true, true))); - TEST_ASSERT_EQUAL_INT(int(GestureEffect::Previous), int(route_gesture(GestureAction::SwipeUp, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::SwipeUp, true, false))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::SwipeDown, true, false))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Wake), int(route_gesture(GestureAction::Wave, true, true))); + TEST_ASSERT_EQUAL_INT(int(GestureEffect::Next), int(route_gesture(GestureAction::Wave, true, false))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::Clock), int(route_gesture(GestureAction::EnterClock, true, false))); TEST_ASSERT_EQUAL_INT(int(GestureEffect::None), int(route_gesture(GestureAction::EnterClock, true, true))); for (auto action : {GestureAction::SwipeLeft, GestureAction::SwipeRight, GestureAction::SwipeUp, - GestureAction::SwipeDown, GestureAction::EnterClock}) + GestureAction::SwipeDown, GestureAction::Wave, GestureAction::EnterClock}) TEST_ASSERT_EQUAL_INT(int(GestureEffect::None), int(route_gesture(action, false, false))); } From 666096bb4c9b3fc53cf4cfbcb58dcd3d7ececce9 Mon Sep 17 00:00:00 2001 From: wulu Date: Fri, 2 Oct 2026 22:08:39 +0800 Subject: [PATCH 09/15] perf(mosaico): shorten startup to first data A cold boot took about 12 s to the first bridge data because the main loop and BLE advertising waited for every peripheral, including about 4.4 s in the battery gauge driver. - Start BLE advertising right after the display so the bridge can reconnect while the remaining peripherals initialize. - Attach the BQ27220 on its own task; the battery stays hidden and the OTA power verdict unavailable until it is ready. - Bring the CO5300 up at zero brightness without the Display On delay and fade in after the first refresh, which removes the white flash of undefined frame memory. The board support change is recorded in UPSTREAM.md. Measured from UART timestamps on one cold boot each: advertising 8.5 s to 3.2 s, main loop 8.5 s to 3.8 s, first time sync 12.1 s to 5.1 s. See docs/validation/mosaico-boot.md. --- docs/validation/mosaico-boot.md | 49 +++++++++++++++++++ .../components/esp-mosaico-bsp/UPSTREAM.md | 6 +++ .../esp-mosaico-bsp/onboard/display.c | 6 ++- .../targets/esp_mosaico/main/app_main.cpp | 40 ++++++++++----- .../targets/esp_mosaico/main/display_ui.cpp | 10 ++-- 5 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 docs/validation/mosaico-boot.md diff --git a/docs/validation/mosaico-boot.md b/docs/validation/mosaico-boot.md new file mode 100644 index 0000000..3cec97f --- /dev/null +++ b/docs/validation/mosaico-boot.md @@ -0,0 +1,49 @@ +# Mosaico 开机耗时 + +用户反馈开机后白屏闪一下，约 10 秒才出现连接、数据和旋转响应。以下时间取自 UART0 启动日志的时间戳（上电起算，单位 ms），调整前后各一次冷启动，均为手势固件、相机开启、Bridge 已配对并在运行。单次测量不代表分布；Bridge 的连接时刻取决于主机端扫描。 + +## 调整前（2026-10-02） + +| 阶段 | 结束时刻 | 耗时 | +|---|---|---| +| 引导程序加载并校验应用（约 5.4 MB） | 989 | 约 960 | +| PSRAM 初始化、代码与只读数据拷入 PSRAM、自检 | 1940 | 约 950，其中自检约 190 | +| 屏幕与触摸初始化 | 3478 | 约 1540 | +| BMI270 | 3885 | 约 410 | +| BQ27220 | 8276 | 约 4390 | +| 按键、BLE 初始化，开始广播 | 8496 | 约 220 | +| 主循环开始 | 8502 | | +| Bridge 连接 | 11008 | | +| 加密完成 | 11607 | | +| 收到对时 | 12094 | | + +- BQ27220：驱动的 `bq27220_seal()` 无条件等待 `CONFIG_BQ27220_SEAL_SETTLE_MS`（2000 ms）。`bq27220_create()` 在校验电池配置后已封存一次，`bsp_battery_init()` 随后再次调用，合计约 4 秒。 +- 白屏：CO5300 初始化序列把亮度寄存器 0x51 设为 0xFF 并发出 Display On，随后等待 600 ms；此时尚未写入任何一帧，面板显示上电后的帧存内容。 +- 主循环在全部外设初始化之后才开始，旋转、渲染和 BLE 事件处理都排在其后。 + +## 调整 + +- 电量计在独立任务中初始化，就绪前电池图标隐藏，OTA 电源判定保持未就绪。封存等待本身未改动。 +- CO5300 以零亮度启动，初始化序列不再包含 Display On 及其 600 ms 等待（`bsp_display_new()` 在设置方向后发出 Display On）；界面在首次刷新完成后经现有的亮度过渡淡入。板级改动记录在 `firmware/components/esp-mosaico-bsp/UPSTREAM.md`。 +- BLE 在屏幕初始化后立即启动，其余外设初始化期间即可被连接；链路事件在队列中等待主循环。 + +## 调整后（2026-10-02） + +| 里程碑 | 调整前 | 调整后 | +|---|---|---| +| 屏幕初始化完成 | 3382 | 2783 | +| 开始广播 | 8496 | 3196 | +| 主循环开始 | 8502 | 3778 | +| Bridge 连接 | 11008 | 4005 | +| 加密完成 | 11607 | 4574 | +| 收到对时 | 12094 | 5059 | +| BQ27220 就绪 | 8276 | 7901 | + +应用通过 ESP-IDF v6.1 构建，按六段写入 ota_0 并全部通过设备哈希校验，未写入 NVS、分区表、引导程序或 OTA 数据。日志中没有新增的错误或告警。白屏是否消除、淡入是否正常、电池图标延后出现的观感，以及电量计与 IMU、触摸并发访问 I2C 的长期稳定性，尚待实机确认。 + +## 未处理 + +- 屏幕初始化中 Sleep Out 之后的 600 ms 等待；`esp_lcd_co5300` 的默认序列为 60 ms，未对照面板模组的上电时序规格。 +- 应用启动前约 1.9 秒：引导程序校验应用、代码与只读数据拷入 PSRAM、PSRAM 自检（约 190 ms）。 +- BMI270 初始化约 400 ms。 +- Bridge 发现并连接设备的耗时在主机端。 diff --git a/firmware/components/esp-mosaico-bsp/UPSTREAM.md b/firmware/components/esp-mosaico-bsp/UPSTREAM.md index eea9067..c7572ac 100644 --- a/firmware/components/esp-mosaico-bsp/UPSTREAM.md +++ b/firmware/components/esp-mosaico-bsp/UPSTREAM.md @@ -28,5 +28,11 @@ and two-pixel vertical boundaries, clipped to the display dimensions. The CO5300 RASET command requires an even start row and an even row count (datasheet page 161); LVGL renders the expanded rows so the pixel buffer matches the transfer window. +The CO5300 initialization sequence sets brightness register 0x51 to zero instead of +0xFF and omits the Display On command with its 600 ms delay. `bsp_display_new()` +issues Display On after the orientation is set. The panel therefore stays dark until +the application raises the brightness, which it must do after the first frame has been +written; upstream shows undefined frame memory at full brightness during that delay. + When re-syncing, preserve these integration changes as well as the manifest constraint and review the repository diff for other board-specific changes. diff --git a/firmware/components/esp-mosaico-bsp/onboard/display.c b/firmware/components/esp-mosaico-bsp/onboard/display.c index ffda2a4..2f544d4 100644 --- a/firmware/components/esp-mosaico-bsp/onboard/display.c +++ b/firmware/components/esp-mosaico-bsp/onboard/display.c @@ -130,11 +130,13 @@ static const co5300_lcd_init_cmd_t s_vendor_init[] = { {0x35, (uint8_t[]){0x00}, 1, 0}, #endif {0x53, (uint8_t[]){0x20}, 1, 0}, - {0x51, (uint8_t[]){0xFF}, 1, 0}, + /* Frame memory is undefined until the first flush, so the panel starts at zero + * brightness; the application raises it once a frame has been written. Display On + * is issued by bsp_display_new() after the orientation is set. */ + {0x51, (uint8_t[]){0x00}, 1, 0}, {0x63, (uint8_t[]){0xFF}, 1, 0}, {0x2A, (uint8_t[]){0x00, 0x00, 0x01, 0xDF}, 4, 0}, {0x2B, (uint8_t[]){0x00, 0x00, 0x01, 0xDF}, 4, 0}, - {0x29, NULL, 0, 600}, }; /* diff --git a/firmware/targets/esp_mosaico/main/app_main.cpp b/firmware/targets/esp_mosaico/main/app_main.cpp index c6be0fd..94865aa 100644 --- a/firmware/targets/esp_mosaico/main/app_main.cpp +++ b/firmware/targets/esp_mosaico/main/app_main.cpp @@ -86,6 +86,20 @@ GestureInput gestures; LinkUpdate ble_link{}; std::atomic status_page{Page::Overview}; std::atomic ai_button_pressed{false}; +PowerMonitor power_monitor; +std::atomic power_monitor_ready{false}; + +// The gauge driver waits out fixed settle times of several seconds while sealing, so the +// gauge attaches on its own task. The battery stays hidden until it is ready. +void power_monitor_task(void* /*context*/) +{ + if (power_monitor.begin()) { + power_monitor_ready.store(true, std::memory_order_release); + } else { + ESP_LOGW(TAG, "BQ27220 telemetry unavailable; battery will be hidden"); + } + vTaskDelete(nullptr); +} const char* get_ui_location(void* context) { @@ -163,6 +177,19 @@ extern "C" void app_main(void) // together with the VCC_3V3 rail, so the panel has to come first. ESP_ERROR_CHECK(ui.begin(initial_rotation) ? ESP_OK : ESP_FAIL); + // Started first so the gauge has attached before BMI270 initialization uses the bus. + if (xTaskCreate(power_monitor_task, "power_init", 4096, nullptr, 1, nullptr) != pdPASS) { + ESP_LOGW(TAG, "BQ27220 start task unavailable; battery will be hidden"); + } + + // Advertising starts before the remaining peripherals so the bridge can reconnect + // while they initialize; link events wait in the queue for the main loop. + ESP_ERROR_CHECK(events.begin() ? ESP_OK : ESP_ERR_NO_MEM); + // Wired before BLE so no peer can reach the manifest path ungated; the + // verdict itself is published by the main-loop power poll below. + ota.set_power_source(&ota_power_source); + ESP_ERROR_CHECK(ble.start(events, ble_config)); + const bool orientation_sensor_ready = (kAutoRotationEnabled || kWakeDetectionEnabled) && init_orientation_sensor(); @@ -187,11 +214,6 @@ extern "C" void app_main(void) } } - PowerMonitor power_monitor; - const bool power_monitor_ready = power_monitor.begin(); - if (!power_monitor_ready) { - ESP_LOGW(TAG, "BQ27220 telemetry unavailable; battery will be hidden"); - } const bool ai_button_ready = init_ai_button(); const PanelSettings persisted_settings = load_panel_settings({ @@ -221,12 +243,6 @@ extern "C" void app_main(void) ScreenRotation applied_rotation = kFixedRotation; uint64_t next_orientation_poll_ms = 0; - ESP_ERROR_CHECK(events.begin() ? ESP_OK : ESP_ERR_NO_MEM); - // Wired before BLE so no peer can reach the manifest path ungated; the - // verdict itself is published by the main-loop power poll below. - ota.set_power_source(&ota_power_source); - ESP_ERROR_CHECK(ble.start(events, ble_config)); - ESP_LOGI(TAG, "ESP-Mosaico usage panel started"); Page page = Page::Overview; @@ -421,7 +437,7 @@ extern "C" void app_main(void) if (now_ms >= next_power_poll_ms) { next_power_poll_ms = now_ms + POWER_POLL_PERIOD_MS; raw_sample = {}; - if (power_monitor_ready) power_monitor.read(raw_sample); + if (power_monitor_ready.load(std::memory_order_acquire)) power_monitor.read(raw_sample); battery_view = power_filter.update(raw_sample, now_ms); const OtaPowerReading reading = derive_power_reading(battery_view, raw_sample); diff --git a/firmware/targets/esp_mosaico/main/display_ui.cpp b/firmware/targets/esp_mosaico/main/display_ui.cpp index 4696d65..4d90bba 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.cpp +++ b/firmware/targets/esp_mosaico/main/display_ui.cpp @@ -152,13 +152,7 @@ lv_display_t* start_display( ESP_LOGE(TAG, "touch initialization failed"); return nullptr; } - const esp_err_t error = bsp_display_brightness_set( - CONFIG_MOSAICO_USAGE_PANEL_DISPLAY_BRIGHTNESS_PERCENT); - if (error != ESP_OK) { - ESP_LOGE(TAG, "display brightness setting failed: %s", - esp_err_to_name(error)); - return nullptr; - } + // The panel comes up at zero brightness; begin() raises it after the first frame. return display; } @@ -380,6 +374,8 @@ bool DisplayUi::begin(ScreenRotation initial_rotation) show_page(Page::Overview, /*animate=*/false); lv_display_add_event_cb( display_, transition_refresh_ready, LV_EVENT_REFR_READY, this); + // Frame memory is undefined until the first refresh, which then fades the panel in. + transition_refresh_pending_ = true; return true; } From 11a3732a52959e8bf239317916e33c9ad98fae2a Mon Sep 17 00:00:00 2001 From: wulu Date: Fri, 2 Oct 2026 22:08:51 +0800 Subject: [PATCH 10/15] fix(mosaico): show battery state without redundant gauge wait - Drop the second bq27220_seal() in bsp_battery_init(). The driver already seals and confirms on every path that returns a handle, and each seal waits 2000 ms unconditionally. The board support change is recorded in UPSTREAM.md. - Colour the battery segments as unavailable when they are created. They used the default theme colour until the first render, which looked like a full battery before the first reading. Measured on one cold boot: gauge ready at 5.9 s instead of 7.9 s. Two other recorded boots found a mismatching stored gauge profile and rewrote it; the cause is not established. See docs/validation/mosaico-boot.md. --- docs/validation/mosaico-boot.md | 13 +++++++++++++ firmware/components/esp-mosaico-bsp/UPSTREAM.md | 5 +++++ .../components/esp-mosaico-bsp/onboard/battery.c | 8 +------- firmware/targets/esp_mosaico/main/display_ui.cpp | 3 +++ 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/validation/mosaico-boot.md b/docs/validation/mosaico-boot.md index 3cec97f..a72ca34 100644 --- a/docs/validation/mosaico-boot.md +++ b/docs/validation/mosaico-boot.md @@ -41,6 +41,19 @@ 应用通过 ESP-IDF v6.1 构建，按六段写入 ota_0 并全部通过设备哈希校验，未写入 NVS、分区表、引导程序或 OTA 数据。日志中没有新增的错误或告警。白屏是否消除、淡入是否正常、电池图标延后出现的观感，以及电量计与 IMU、触摸并发访问 I2C 的长期稳定性，尚待实机确认。 +## 去掉重复封存（2026-10-02） + +`bsp_battery_init()` 不再在 `bq27220_create()` 之后再次封存；驱动在返回句柄的两条路径上都已封存并确认。应用通过 ESP-IDF v6.1 构建，按六段写入 ota_0 并全部通过设备哈希校验。 + +| 冷启动 | 配置核对 | BQ27220 就绪 | +|---|---|---| +| 烧录后第一次 | 首项 `gauging_config`（0x929B）预期 3345、读到 4138，驱动重写全部配置并重新初始化 | 10217 | +| 第二次 | 核对通过，跳过更新 | 5890 | + +第一次启动中用户观察到电池显示先满、再空、后恢复；日志未打印电量值，推断为电量计重新初始化期间的读数。重写包含学习容量，电量计学到的满充容量已回到默认 80 mAh。 + +随后一次烧录后的第一次启动再次出现相同的不一致（同一字段、同样读到 4138）并重写，就绪时刻 10220；其后的普通重启核对通过，就绪时刻 5888。已记录的启动中，两次不一致都发生在经过 ROM 下载模式烧录后的第一次启动，普通重启均核对通过，包括紧接在一次核对通过的后台初始化启动之后的重启。这不支持“后台初始化导致下一次启动读错”，当时看来与“设备停留在下载模式期间电量计配置恢复为其他值”一致；机制未确认，后续记录见“电池显示提前”一节。更早一次记为“烧录后核对通过”的启动之前，设备已自行离开下载模式，可能存在未记录的启动。驱动读取配置时不校验校验和。每次重写都会把学习到的满充容量恢复为默认 80 mAh。 + ## 未处理 - 屏幕初始化中 Sleep Out 之后的 600 ms 等待；`esp_lcd_co5300` 的默认序列为 60 ms，未对照面板模组的上电时序规格。 diff --git a/firmware/components/esp-mosaico-bsp/UPSTREAM.md b/firmware/components/esp-mosaico-bsp/UPSTREAM.md index c7572ac..cd830a1 100644 --- a/firmware/components/esp-mosaico-bsp/UPSTREAM.md +++ b/firmware/components/esp-mosaico-bsp/UPSTREAM.md @@ -34,5 +34,10 @@ issues Display On after the orientation is set. The panel therefore stays dark u the application raises the brightness, which it must do after the first frame has been written; upstream shows undefined frame memory at full brightness during that delay. +`bsp_battery_init()` does not seal the BQ27220 again after `bq27220_create()`. The +driver seals and verifies on every path that returns a handle, and each seal waits +`CONFIG_BQ27220_SEAL_SETTLE_MS` (2000 ms) unconditionally, so the upstream call only +added that wait. + When re-syncing, preserve these integration changes as well as the manifest constraint and review the repository diff for other board-specific changes. diff --git a/firmware/components/esp-mosaico-bsp/onboard/battery.c b/firmware/components/esp-mosaico-bsp/onboard/battery.c index d6c0fe7..027fb41 100644 --- a/firmware/components/esp-mosaico-bsp/onboard/battery.c +++ b/firmware/components/esp-mosaico-bsp/onboard/battery.c @@ -86,14 +86,8 @@ static esp_err_t battery_attach(void) .cedv = &s_default_cedv, }; s_gauge = bq27220_create(&config); + /* bq27220_create() returns a handle only after sealing the gauge and confirming it. */ ESP_RETURN_ON_FALSE(s_gauge, ESP_FAIL, TAG, "attach BQ27220 0x%02X failed", BSP_BATTERY_I2C_ADDR); - const esp_err_t seal_ret = bq27220_seal(s_gauge); - if (seal_ret != ESP_OK) { - ESP_LOGE(TAG, "seal BQ27220 after initialization failed: %s", esp_err_to_name(seal_ret)); - bq27220_delete(s_gauge); - s_gauge = NULL; - return seal_ret; - } const uint16_t voltage_mv = bq27220_get_voltage(s_gauge); ESP_LOGI(TAG, "BQ27220 online: address=0x%02X SDA=%d SCL=%d %u mV", BSP_BATTERY_I2C_ADDR, diff --git a/firmware/targets/esp_mosaico/main/display_ui.cpp b/firmware/targets/esp_mosaico/main/display_ui.cpp index 4d90bba..63a2577 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.cpp +++ b/firmware/targets/esp_mosaico/main/display_ui.cpp @@ -465,6 +465,9 @@ void DisplayUi::build_header(lv_obj_t* screen) battery_segments_[i] = lv_obj_create(body); lv_obj_set_pos(battery_segments_[i], 2 + i * 6, 2); lv_obj_set_size(battery_segments_[i], 5, 8); + // The first frame precedes the first battery reading and shows it as unavailable. + lv_obj_set_style_bg_color(battery_segments_[i], lv_color_hex(COLOR_BATTERY_OFF), 0); + lv_obj_set_style_bg_opa(battery_segments_[i], LV_OPA_COVER, 0); lv_obj_set_style_border_width(battery_segments_[i], 0, 0); lv_obj_set_style_radius(battery_segments_[i], 1, 0); lv_obj_set_style_pad_all(battery_segments_[i], 0, 0); From eb7c402cd50a05791a322d7ea1b64b4733bf4a94 Mon Sep 17 00:00:00 2001 From: wulu Date: Fri, 2 Oct 2026 22:08:51 +0800 Subject: [PATCH 11/15] perf(mosaico): shorten BQ27220 seal settle time Set CONFIG_BQ27220_SEAL_SETTLE_MS to 200 instead of the driver default of 2000 so the first battery reading is not held back by the wait. The driver still confirms the sealed state after the wait and fails initialization otherwise. 200 ms is a trial value without datasheet support. On the one boot recorded with it the seal confirmation passed, but the stored gauge profile mismatched again and was rewritten, and later recordings lost the gauge output. Whether this value affects the stored profile is not established; revert this commit alone to restore the default. --- docs/validation/mosaico-boot.md | 14 ++++++++++++++ firmware/targets/esp_mosaico/sdkconfig.defaults | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/docs/validation/mosaico-boot.md b/docs/validation/mosaico-boot.md index a72ca34..d680c7f 100644 --- a/docs/validation/mosaico-boot.md +++ b/docs/validation/mosaico-boot.md @@ -60,3 +60,17 @@ - 应用启动前约 1.9 秒：引导程序校验应用、代码与只读数据拷入 PSRAM、PSRAM 自检（约 190 ms）。 - BMI270 初始化约 400 ms。 - Bridge 发现并连接设备的耗时在主机端。 + +## 电池显示提前（2026-10-02） + +用户反馈电池显示出现得慢；其他启动项不再调整。 + +- 电池格在创建时即按“无数据”着色。此前未设置颜色，第一帧显示为界面库默认的浅色，看起来像满格，随后依次变为空、真实电量。 +- `CONFIG_BQ27220_SEAL_SETTLE_MS` 由驱动默认的 2000 改为 200。驱动在等待后仍读取状态确认已封存，未封存时初始化失败、电池隐藏。200 ms 没有数据手册依据，属试验值。 +- 电量计任务在屏幕初始化后立即启动，先于 BLE 和 BMI270 初始化。 + +应用通过 ESP-IDF v6.1 构建，按六段写入 ota_0 并全部通过设备哈希校验。烧录结束后设备自行离开下载模式并启动，该次启动未被记录。 + +其后记录到一次普通重启：电量计初始化在 3004 ms 开始，核对再次在同一字段读到 4138，驱动重写配置，就绪时刻 7595；200 ms 等待后的封存确认通过。这次启动不是烧录后的第一次启动，此前应用已连续运行至少四分钟，因此上文“不一致只出现在经过下载模式之后”的推断不成立。随后两次快速重启的 UART 记录大量丢失，没有采到电量计日志，无法判断。 + +未确认事项：200 ms 等待是否会使电量计在下一次启动时配置异常；配置不一致与关机时长的关系；普通启动中电量计的实际就绪时刻。若确认与 200 ms 有关，恢复 `CONFIG_BQ27220_SEAL_SETTLE_MS` 的默认值即可，电量计任务提前启动可以保留。 diff --git a/firmware/targets/esp_mosaico/sdkconfig.defaults b/firmware/targets/esp_mosaico/sdkconfig.defaults index 67a6219..0482530 100644 --- a/firmware/targets/esp_mosaico/sdkconfig.defaults +++ b/firmware/targets/esp_mosaico/sdkconfig.defaults @@ -12,6 +12,10 @@ CONFIG_SPIRAM_MODE_OCT=y CONFIG_SPIRAM_SPEED_250M=y CONFIG_SPIRAM_XIP_FROM_PSRAM=y +# The driver default of 2000 ms delays the first battery reading; the driver still +# confirms the sealed state after this wait and fails initialization otherwise. +CONFIG_BQ27220_SEAL_SETTLE_MS=200 + CONFIG_BSP_DISPLAY_ENABLE=y CONFIG_BSP_CO5300_ENABLE_TE=y From 2fd32ec269fc4f687934014d51d2a0537a1284e4 Mon Sep 17 00:00:00 2001 From: wulu Date: Fri, 2 Oct 2026 22:45:28 +0800 Subject: [PATCH 12/15] docs(mosaico): clarify experimental gestures and code comments --- README.md | 2 +- README.zh-CN.md | 2 +- THIRD_PARTY_LICENSES.md | 1 + docs/design/amoled-ui.md | 10 +- docs/design/mosaico-gesture-input.md | 195 +++++------------- docs/devices.md | 6 +- docs/roadmap.md | 2 +- docs/validation/mosaico-boot.md | 80 ++----- docs/validation/mosaico-gestures.md | 171 +++------------ .../components/esp-mosaico-bsp/UPSTREAM.md | 5 +- .../targets/esp_mosaico/CAMERA_UPSTREAM.md | 12 +- firmware/targets/esp_mosaico/README.md | 15 +- .../targets/esp_mosaico/main/app_main.cpp | 13 +- .../esp_mosaico/main/gesture_image.hpp | 26 ++- .../esp_mosaico/main/gesture_input.cpp | 4 +- .../esp_mosaico/main/gesture_tracker.cpp | 31 ++- .../esp_mosaico/main/gesture_tracker.hpp | 4 +- 17 files changed, 169 insertions(+), 410 deletions(-) diff --git a/README.md b/README.md index 96f601b..c983170 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The data source handles account sign-in and usage retrieval. Bridge sends only u | Waveshare AMOLED 2.16 | 480×480 AMOLED | Touchscreen, clock screensaver, auto-rotation | | M5StickS3 | 135×240 LCD | Button navigation, compact size | | Waveshare ePaper 3.97 | 800×480 e-paper with four grayscale levels | Temperature and humidity, usage trends (microSD card required) | -| ESP-Mosaico | 480×480 AMOLED | Touchscreen, AI button, auto-rotation | +| ESP-Mosaico | 480×480 AMOLED | Touchscreen, AI button, auto-rotation, camera gestures (experimental) | | Waveshare RLCD 4.2 | 400×300 monochrome reflective LCD | KEY / BOOT navigation, clock, usage trends (microSD card required) | All devices support Codex / Claude usage display, retain the last usage data when disconnected, and support firmware updates over Bluetooth. See the [device guide](docs/devices.md) for a full feature comparison and control shortcuts. diff --git a/README.zh-CN.md b/README.zh-CN.md index c806834..93cbf2e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -30,7 +30,7 @@ flowchart LR | Waveshare AMOLED 2.16 | 480×480 AMOLED | 触屏、时钟屏保、自动旋转 | | M5StickS3 | 135×240 LCD | 按键翻页、小尺寸 | | Waveshare ePaper 3.97 | 800×480 四灰阶墨水屏 | 温湿度、用量趋势（需 microSD 卡） | -| ESP-Mosaico | 480×480 AMOLED | 触屏、AI 键、自动旋转 | +| ESP-Mosaico | 480×480 AMOLED | 触屏、AI 键、自动旋转、摄像头手势（实验性） | | Waveshare RLCD 4.2 | 400×300 黑白反射式 LCD | KEY / BOOT 翻页、时钟、用量趋势（需 microSD 卡） | 所有设备均支持 Codex / Claude 用量显示、断连保留最后用量与蓝牙固件更新。完整功能对照及快捷操作见[设备说明](docs/devices.md)。 diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index 7c2f032..5f11724 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -173,6 +173,7 @@ For ESP Component Registry entries, `Registry license` is the API value for the - Source: https://components.espressif.com/components/espressif/usb/versions/1.3.0 - SPDX: `Apache-2.0` - Copyright/notice: https://github.com/espressif/esp-usb/blob/ded385702ecae4d5c15c497149062ce577fa167a/host/usb/LICENSE +- ESP-Mosaico version: `1.5.0`; source: https://components.espressif.com/components/espressif/usb/versions/1.5.0; copyright/notice: https://github.com/espressif/esp-usb/blob/eb4618660a83e80cd57dcda736edbfa884cb88cc/host/usb/LICENSE ## `idf` diff --git a/docs/design/amoled-ui.md b/docs/design/amoled-ui.md index 2b3a8c7..baa3325 100644 --- a/docs/design/amoled-ui.md +++ b/docs/design/amoled-ui.md @@ -72,19 +72,19 @@ Waveshare：屏幕边距 24px；卡片宽 432px；卡片内边距 20–22px； - **底部导航**：方屏为**纯文字标签 + 下划线指示**，不用色块、不用卡片底。34px 高的三等分标签条，标签 13px 600 大写、字距 0.06em；未选中 `#565D68`，选中 `#C7CCD4` 并在标签下方 7px 处加一条 26×2px（圆角 1px）`#C7CCD4` 短横。导航是次要信息，任何情况下都不能成为屏幕上最亮的元素。**下划线是三个标签共用的同一条**，翻页时以 180ms 缓出横向滑到新标签下方，而不是在旧位置熄灭、新位置点亮；页面本身仍是硬切换。 - **连接状态**：右上角一个 8px 状态色圆点 + 14px `#C7CCD4` 文字。不显示 `SECURE` 之类无操作价值的标签。 - **电池**：连接状态右侧，30×14 描边（1px `#565D68`）+ 2px 正极点，内含 4 段。**分级显示，不显示百分比**：FULL=4 段、HIGH=3 段、MID=2 段、LOW=1 段、无数据=0 段；点亮色 `#C7CCD4`，熄灭段 `#2C313B`；仅 LOW 时点亮段转 `#F5B84C`。**充电**：外框左侧 gap 5px 有 7×12 闪电槽，**始终占位**（不充电时留空，绝不收起，否则插拔 USB 会让 `LINKED` 平移）；充电时填 `#C7CCD4`，段数照常反映电量，不做动画；充电时 LOW 不再变黄（外部供电已接入，此时低电量告警不再提供操作价值），屏保 `LOW` 角标同时撤掉。USB 只是供电，不改变 `LINKED`，界面不出现 `USB` 标签。 -- **摄像头标识**（仅 Mosaico）：`CAMERA` 开关已保存为打开时，连接状态点左侧 10px 显示 26×16 实心相机剪影（顶部 8×4 取景凸起，bg 色镜头环内含 4px 镜头点），与状态点垂直居中。采集中（启动或识别）为 `text` 色；开关打开但采集暂停（设置、配对、OTA）为 `dim`；采集失败时为 `dim` 并叠加 `#C7CCD4` 斜杠（外圈 bg 色描边）。开关关闭时不占位。顶栏不显示手势文字提示，仅保留 OK 停留进度条。保持灰阶，不使用 good/warn/bad 色。屏保不显示。 +- **摄像头标识**（仅 Mosaico，实验性手势）：`CAMERA` 开关已保存为打开时，连接状态点左侧 10px 显示 26×16 实心相机剪影（顶部 8×4 取景凸起，bg 色镜头环内含 4px 镜头点），与状态点垂直居中。采集中（启动或识别）为 `text` 色；开关打开但采集暂停（设置、配对、OTA）为 `dim`；采集失败时为 `dim` 并叠加 `#C7CCD4` 斜杠（外圈 bg 色描边）。开关关闭时不占位。顶栏不显示手势文字提示，仅保留 OK / 点赞停留进度条。保持灰阶，不使用 good/warn/bad 色。屏保不显示。 - **相机预览**（仅 Mosaico）：识别就绪且处于普通页面时，点击顶栏摄像头标识切换预览，再次点击关闭；每次启动默认隐藏，不持久化。浮层位于内容区右上角，距右侧 24px、距内容区顶部 8px；按画面比例显示，最长边 160px，不裁切、不拉伸。只显示相机图像及 2px `text` 色手部定位框，无手时隐藏定位框，不叠加检测数量、置信度或状态文字。开关只控制预览显示，手势识别由设置页 CAMERA 控制。时钟、设置、配对、OTA、相机暂停或故障时隐藏浮层；识别恢复且返回普通页面后恢复本次运行的预览选择。 - **时间**：顶栏左侧，22px 700，`text` 色，24 小时制不显示秒。页面归属由底部导航指示，无需重复标题。硬件无温湿度传感器，界面不出现环境信息。 - **屏保态（SCREENSAVER）**：空闲 5 分钟后进入，触摸屏幕或晃动（IMU）唤醒，回到进入前的页面。规则是**整体降一级**——时钟 `muted #8B93A1`（120px，居中 top 158）、日期一行 20px `dim`、顶栏与底部导航全部隐藏、卡片去掉 panel 底与描边（纯 `bg`，AMOLED 省电）。屏幕上不得出现 `text` 级亮度。底部 13px `#2C313B` 提示 `TAP OR SHAKE TO WAKE`，进入 3 秒后淡出。仅电池 LOW 时右上角显示降亮的 `LOW` + 电池图标（`#8A6A2E`），其余状态一概不显示。防烧屏：整块内容每 60 秒随机位移 ±10px。 ## 6. 本地显示设置 -微雪设备长按触屏约 0.8 秒进入全屏设置页。亮度范围 1–100%，调整时预览；自动进入时钟支持 OFF、1、5、10、30 分钟。SAVE 写入 NVS，CANCEL 恢复已保存值；首次启动使用 menuconfig 默认值。OFF 不影响主动切换时钟。设置页打开时保持唤醒，配对和 OTA 优先于设置并撤销未保存预览。 +微雪 AMOLED 2.16 与 ESP-Mosaico 长按触屏约 0.8 秒进入全屏设置页。亮度范围 1–100%，调整时预览；自动进入时钟支持 OFF、1、5、10、30 分钟。SAVE 写入 NVS，CANCEL 恢复已保存值；首次启动使用 menuconfig 默认值。OFF 不影响主动切换时钟。设置页打开时保持唤醒，配对和 OTA 优先于设置并撤销未保存预览。 -设置页采用 24px 边距的行式布局，两项设置以 1px `#1D2128` 分隔线分组，不增加卡片底色与描边。顶栏左侧显示 DISPLAY，右侧复用连接状态与电池；LINKED 保持绿色。设置操作优先保证近距离触控，不要求全部读数使用远距离展示字号。 +设置页采用 24px 边距的行式布局，设置项以 1px `#1D2128` 分隔线分组，不增加卡片底色与描边。顶栏左侧显示 DISPLAY，右侧复用连接状态与电池；LINKED 保持绿色。设置操作优先保证近距离触控，不要求全部读数使用远距离展示字号。 -- **亮度设置行**：复用现有 48px 数值字体；右侧加减按钮为 80×80px，间隔 8px。仅通过按钮调整，不显示不可操作的滑条。到达 1% 或 100% 时禁用对应按钮，边界不循环。 -- **有限选项选择**：AUTO CLOCK 下方标明 AFTER IDLE / MIN；OFF、1、5、10、30 五项为 80×64px，间隔 8px，一次点击选定。只有当前选项反色，不累计填充；OFF 不拼接 MIN。保留非标准 menuconfig 默认时长，未匹配预设时显示当前秒数，直到用户主动选择预设。 +- **亮度设置行**：复用现有 48px 数值字体；右侧加减按钮为 80×80px，Mosaico 为 80×72px，间隔 8px。仅通过按钮调整，不显示不可操作的滑条。到达 1% 或 100% 时禁用对应按钮，边界不循环。 +- **有限选项选择**：AUTO CLOCK 下方标明 AFTER IDLE / MIN；OFF、1、5、10、30 五项为 80×64px，Mosaico 为 80×52px，间隔 8px，一次点击选定。只有当前选项反色，不累计填充；OFF 不拼接 MIN。保留非标准 menuconfig 默认时长，未匹配预设时显示当前秒数，直到用户主动选择预设。 - **操作状态**：16px 圆角，可见边界与点击区域一致。选中项及可用 SAVE 使用 text 底、bg 字；未选项和 CANCEL 使用 panel 底、muted 字。按下时背景和边框产生灰阶反馈；禁用控件使用 panel 底、dim 字，不响应点击。不使用主题阴影或过渡。 - **开关**（仅 Mosaico `CAMERA`）：独立一行，与上方以分隔线隔开；右侧 80×44px 胶囊开关，与加减按钮右对齐。关闭为 panel 底、muted 圆钮居左；打开为反色 text 底、bg 圆钮居右。与亮度、自动时钟一样只是预览：切换后启用 SAVE，SAVE 一并写入 NVS，CANCEL 恢复已保存值。 - **保存与取消**：底部按钮均为 206×64px，间隔 20px。未修改时禁用 SAVE，修改后启用；CANCEL 始终可用。保存成功后退出，失败时保留页面和预览，在按钮上方显示 SAVE FAILED - TRY AGAIN 并允许重试。不常驻显示持久化实现说明。 diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md index e0ecc61..3c92aa9 100644 --- a/docs/design/mosaico-gesture-input.md +++ b/docs/design/mosaico-gesture-input.md @@ -1,179 +1,82 @@ -# Mosaico 本地手势输入规划 +# Mosaico 实验性手势输入 -状态：软件实现与构建验证完成，待实机验收。2026-09-22 核对；工程基线 `07ac386364ff680bf1835ace646c788975a0cbbb`，规划分支 `codex/mosaico-gesture-plan`。 +本功能使用左槽 CameraBoard 在设备本地识别手势，默认关闭。用户操作见[使用与安装](../validation/mosaico-gestures.md)，固定依赖与参考仓库见[摄像头依赖说明](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)。 -## 目标与范围 +## 交互 -在 ESP-Mosaico 上增加可选的本地摄像头输入，保留触摸、AI 键、BLE 和自动时钟屏保。先验证 CameraBoard → HandDetect → 四向轨迹 → 翻页/唤醒，再增加现成分类器的 `ok` 手势进入时钟。 - -摄像头图像仅在设备内处理，不上传、不默认保存、不显示常驻预览。第一阶段不训练模型、不引入 ESP-VISION/MicroPython 或 ESP-WHO 整套应用、不扩展 Bridge 协议、不把视觉依赖加入其他板卡。识别区域内出现手不等于用户发出命令。 - -此处“解锁”指退出时钟屏保，不是身份验证，也不涉及芯片深度睡眠唤醒。摄像头唤醒要求采集与识别链路仍在运行，不能据此承诺低功耗。 - -## 两份报告的取舍 - -已阅读分享对话和补充报告。采用分享对话最后确认的“挥手翻页/唤醒”语义；👌 OK 进入时钟作为第二阶段能力。空中挥手按导航项方向切页，触摸滑动保持原有语义。 - -| 当前状态 | 有效动作 | 行为 | -| --- | --- | --- | -| 正常用量页 | 向左挥手 | 上一页 | -| 正常用量页 | 向右挥手 | 下一页 | -| 正常用量页 | 向上挥手 | 上一页 | -| 正常用量页 | 向下挥手 | 下一页 | -| 时钟屏保 | 任一有效四向挥手 | 只恢复原页面；本轮不再翻页 | -| 正常用量页 | 稳定 `ok` | 进入时钟并保存原页面 | -| 时钟屏保 | `ok` | 无操作，不执行 Toggle | -| 设置、配对确认、OTA | 任意手势 | 丢弃，不延后执行 | - -上下左右方向均为用户面对当前屏幕所见的方向。挥动判定只用 HandDetect，因此不能宣称识别了“张掌”；若日常活动误触发无法达标，再评估以 `five` 分类作为挥动的手形门控。 - -补充报告中“看到手/普通运动即可唤醒”暂不采纳，避免桌前活动持续打断屏保。ROI motion 仅是后续功耗优化候选，不能代替有效挥手确认。 - -## 已核对的证据与缺口 - -| 项目 | 当前证据 | 对实施的影响 | +| 当前页面 | 动作 | 行为 | | --- | --- | --- | -| ESP-DL S31 支持 | Model Zoo 列出 S31；HandDetect 与 HandGesture 的 CMake 均把 `esp32s31` 映射到 `models/p4` | 发布组件已在 IDF v6.1 下完整构建通过；真实推理仍待实机验证 | -| 现成动作类别 | 示例列出 `five`、`ok` 等类别，没有独立 `fist`；`no_gesture` 是未定义手势 | 不把 `no_gesture` 当作拳头，不投入自训练 | -| 模型输入与性能 | 检测输入 224×224，分类输入 128×128；子目录耗时表只给 S3/P4 | 不将 P4 耗时外推为 S31 性能，也不将输入大小视为相机输出模式 | -| ESP-VISION | 官方支持页列出 ESP32-S31-MOSAICO；README 指定 S31 使用 IDF v6.1 | 参考 board port、采集和转换；本次未独立确认报告中的 CI 配置细节 | -| ESP-WHO | README 描述采集/推理异步运行；板卡清单未列 Mosaico | 仅作生命周期与任务划分参考 | -| 当前 BSP | `BSP_CAPS_CAMERA` 为 0，已有扩展槽电源/I2C/GPIO 基础，无采集组件依赖 | CameraBoard 接入是首个工程任务，不能只改能力宏 | -| 摄像头参考实现 | 相邻 esp-mosaico-claw 工程有 OV3640 DVP 的 `mosaico_camera`，当前接口支持左槽和帧获取/归还 | 它依赖 `subboard_support`、模块管理、board manager 及 esp-claw 内的 esp_video；不能直接复制一个组件就认为可用 | -| 固件存储 | 目标配置 16 MB Flash；两个 OTA app 分区各 7 MiB | 完整应用约 5.16 MiB，模型随应用更新 | +| 用量页 | 任一方向挥手或左右招手 | 下一页，按总览 → Codex → Claude 循环 | +| 时钟 | 挥手或左右招手 | 只恢复原页面，本轮不翻页 | +| 用量页 | 稳定 OK 或点赞约 500 ms | 进入时钟并保留原页面 | +| 时钟 | OK 或点赞 | 不切换页面 | +| 设置、配对确认、OTA | 任意手势 | 不接受动作，不延后执行 | -相邻工程核对时 HEAD 为 `903fe11ec3ca4a8881afef104e53a8da040896ac`，摄像头头文件和实现存在未提交修改，README 的默认分辨率/缓冲数量也与当前头文件不同。它只提供参考，实施时必须选定可复现的来源和许可，不能将本地工作区当作已验证的上游版本。 +触摸滑动保留双向导航。摄像头唤醒依赖持续采集和识别，时钟屏保不是芯片深度睡眠。 -## 现有接入点 +设置页的 `CAMERA` 开关与亮度、自动时钟超时一并保存。调整开关只改变设置预览，`SAVE` 成功后才改变采集状态，`CANCEL` 恢复保存值。NVS 使用 `mosaico/settings` 单个整数保存三个字段；记录缺失或无效时使用默认显示设置并关闭相机。 -路径均相对仓库根目录。 +图像仅在本机处理，不上传或保存。点击摄像头标识切换预览，预览选择不持久化。界面尺寸与状态见[AMOLED 设计规范](amoled-ui.md)。 -| 路径/符号 | 复用方式 | -| --- | --- | -| `firmware/targets/esp_mosaico/main/app_main.cpp` | 主循环统一拥有页面、屏保、设置、OTA/配对门控；新增手势消费点 | -| `firmware/targets/esp_mosaico/main/orientation.cpp` / `navigation_swipe()` | 现有左滑→Next、右滑→Previous；保留触摸映射，单独校准相机→屏幕坐标 | -| `firmware/components/usage_panel_state/include/usage_panel_state/page_state.hpp` / `page_after_swipe()` | 复用页序与边界行为 | -| `firmware/components/usage_panel_state/screensaver_state.cpp` | 复用原页面保存、活动时间和屏保生命周期；按需要增加明确的输入方法 | -| `firmware/targets/esp_mosaico/main/display_ui.hpp` / `request_page()` | 主循环同步渲染请求，不由推理任务操作 LVGL | -| `firmware/components/usage_ble/include/usage_ble/app_events.hpp` | 当前队列是 BLE 边界，不加入摄像头帧或视觉依赖 | -| `firmware/components/esp-mosaico-bsp/include/bsp/subboard.h` | 复用扩展槽资源；确认 GPIO14 在摄像头模式是 D4，不能同时作 EEPROM 地址选择输出 | -| `firmware/components/esp-mosaico-bsp/UPSTREAM.md` | 保留已有 IDF 约束与 TE 局刷修改；必要的 BSP 修改记录于此 | - -不能直接调用现有 `ScreenPage` 远程事件实现手势：当前远程分支在屏保中会恢复页面并继续翻页，还会关闭设置。手势需要“唤醒即消费”和“设置中忽略”的独立策略。 - -## 最小实现结构 +## 处理链路与所有权 ```text CameraBoard / esp_video - → 获取最新帧、格式转换 - → HandDetect - → 屏幕坐标中的单手轨迹 - → SwipeLeft / SwipeRight / SwipeUp / SwipeDown / EnterClock - → Mosaico 主循环检查状态 - → ScreensaverController / page_after_swipe / ui.request_page + → 最新 UYVY 帧 + → 等比例 RGB888 转换、镜像与旋转 + → HandDetect → 单手轨迹 / HandGestureCls + → SwipeLeft / SwipeRight / SwipeUp / SwipeDown / Wave / EnterClock + → Mosaico 主循环检查上下文 + → 翻页 / 唤醒 / 进入时钟 ``` -- 一个视觉 worker 串行完成采集、推理和轨迹更新即可起步；先测量再决定是否拆任务或绑核。模型不在 UI 主循环或 LVGL 锁内运行。 -- 只在 Mosaico 增加相机适配和纯轨迹判定代码，优先放目标目录；只有需要独立管理第三方来源时才抽出组件，不预建跨板卡视觉框架。 -- 相机帧的获取与归还由 worker 拥有。所有成功、失败、停用路径归还帧；停止时先结束当前推理并归还缓冲，再关闭流和释放资源。结果中不保留驱动/模型内部容器的借用引用。 -- 采用有界的小事件通道，初始可只有一个待消费事件；满时丢弃，不阻塞。事件包含动作、时间戳和输入会话代次，过期或上下文已改变的事件丢弃。 -- 主循环在处理最新 BLE/OTA/设置状态后消费手势。屏保、页面或输入许可变化时更新会话代次，取消旧轨迹；关闭功能、旋转、错误恢复也清空待执行动作。 -- 首次挥手若发生在屏保中，只唤醒并同步 `page`、`requested_page` 与状态上报；不能随后落入翻页分支。只在命令被接受时重置活动计时。 -- 第二阶段为进入时钟提供幂等语义，不重复调用 `note_remote_toggle()`。如需公共方法，只增加最小的明确进入/有效输入唤醒接口，保持其他输入行为。 - -### 轨迹与一次性触发 - -使用时间戳而非固定帧数定义窗口；检测帧率下降不应改变动作语义。bbox 中心先转为归一化屏幕坐标，包含相机安装方向、镜像与当前已应用的屏幕旋转。旋转过程中放弃当前动作。 - -挥手状态为 `Idle → Tracking → Cooldown → Idle`；OK 及上下文失效另需 `WaitRelease`： - -1. 单个可信目标稳定出现才开始跟踪；以位置连续性和 bbox 尺寸约束保持目标关联。 -2. 在限定时间内检查主轴净位移、累计路径方向一致性、垂直于主轴的漂移、置信度与合理 bbox 大小。突然跳变、长时间漏检、模糊的多手关联取消候选。 -3. 完成一次动作只发一个事件。挥手后经过冷却即可再次挥手，手不必离开；收手窗口内的反向挥动视为收手并忽略。OK、多手及与挥手无关的上下文失效之后，等待持续无手且最短冷却已结束再重新接受输入。 -4. 单帧漏检、分类器 `no_hand` 或 worker 暂停不能单独作为“手已离开”的证据。恢复采集后要观察新鲜帧，禁止旧事件回放。 - -以下仅为试验起点，不是已验证的产品参数：主轴净位移约对应画面宽度或高度的 25%，动作窗口 150–900 ms，主轴净位移至少为另一轴最大偏移的 2 倍；连续无手约 300 ms，最短冷却约 500 ms，收手窗口约 1.5 s，帧间隔上限约 400 ms；事件有效期约 300 ms。阈值结合有效帧数、视场、距离和漏检率一起校准。持续可见的手不因冷却超时而重复翻页。 - -## 分阶段工作与完成条件 +| 文件 | 职责 | +| --- | --- | +| `firmware/targets/esp_mosaico/main/gesture_camera.cpp` | 左槽引脚、V4L2 流、帧获取与归还、停止及资源释放 | +| `firmware/targets/esp_mosaico/main/gesture_image.cpp` | UYVY 转 RGB888、等比例缩放、镜像和旋转 | +| `firmware/targets/esp_mosaico/main/gesture_tracker.cpp` | 单手轨迹、招手、姿态停留与重复触发抑制 | +| `firmware/targets/esp_mosaico/main/gesture_input.cpp` | 视觉任务、模型生命周期、上下文和有界事件队列 | +| `firmware/targets/esp_mosaico/main/app_main.cpp` | 设置、页面、屏保和输入许可；消费手势并更新 UI | +| `firmware/targets/esp_mosaico/main/display_ui.cpp` | 摄像头标识、停留进度和预览；LVGL 访问受显示锁保护 | +| `firmware/targets/esp_mosaico/main/panel_settings.cpp` | 显示与相机设置的原子持久化 | -### P0：摄像头与模型可行性 +视觉任务串行执行采集、检测、分类和轨迹更新。V4L2 帧转换后立即归还；模型不在主循环或 LVGL 锁内运行。预览回调从视觉任务调用，取得显示锁后复制像素；隐藏时跳过显示锁和绘制。 -- [ ] 核对实物主板版本、OV3640 CameraBoard、左槽安装方向；确认扩展槽电源、共享 I2C 和 GPIO 复用。 -- [ ] 固定 esp_video、esp_cam_sensor、HandDetect/ESP-DL 的可获取版本或提交和模型摘要；核查发布包确实含 S31 路径，而不只检查 master。现有 master manifest 的版本为 hand_detect 0.2.0、ESP-DL `~3.3.0`，不能因此认定任意同版本包都含相同支持。 -- [ ] 在现有 BSP 上接入最小采集，避免带入整个模块管理/应用调度框架；如必须复用其一部分，记录具体资源所有权和依赖理由。 -- [ ] 获取实际协商的像素格式、宽高、stride 和缓冲数量；验证 RGB565 字节序或 UYVY 转换、缩放/裁剪和 bbox 坐标。不要直接把 UYVY 指针作为 RGB 图像输入。 -- [ ] 先运行静态图 HandDetect，再接实时帧；确认推理结果和帧归还；在现有 UI、BLE、IMU、电量轮询共存时采集耗时和内存。 -- [ ] 核算最终 app 大小、模型大小、内部 RAM/PSRAM 低水位及连续块。224×224 RGB888 输入约 147 KiB；640×480 双缓冲 RGB565 约 1.17 MiB，仅是缓冲示例，不含张量、显示、驱动和栈，也不表示该格式已被硬件接受。 +主循环拥有页面和屏保状态。事件队列容量为 1，满时丢弃，不阻塞推理。每个事件带时间戳和上下文代次；超过 300 ms 或代次不匹配时丢弃。触摸、按键、连接变化、页面、旋转和输入许可变化使旧轨迹失效。被接受的手势会保留自身的重复触发抑制状态。 -存储决策：模型内嵌 app，使模型随 A/B OTA 和回滚一起切换。裁减未使用的 ESP-DL 像素转换后，完整应用约 5.16 MiB，采用 7 MiB 双槽。NVS、PHY 与 otadata 使用当前分区表指定的地址。设备使用该分区表后，可通过 OTA 更新应用与模型；完整安装和实机验证见[固件安装与验收](../validation/mosaico-gestures.md)。 +关闭相机或进入受保护状态时，视觉任务完成当前帧后停止流并释放模型和缓冲。启动失败最多重试三次；连续三次采集失败停用识别。普通故障可通过关闭并重新开启重试；无法安全释放驱动资源时，需重启设备。主循环的触摸、按键和 BLE 路径独立于视觉任务。 -**完成条件：**可复现的依赖与构建、可用的实时 bbox、资源与延迟记录、正常停止/缺摄像头降级。只构建成功不算实时识别通过。当前已完成软件接入及 UI，以便一次烧录完成端到端验收；实时识别和资源指标未通过实机验证前，不视为产品验收完成。 +## 识别参数 -### P1：四向挥手翻页与屏保唤醒 +以下为当前实验性实现的参数，集中定义于 `gesture_tracker.cpp`，不代表识别率或响应时间承诺。 -- [ ] 完成纯轨迹判定和坐标映射；原型先验证固定朝向，再验证四个朝向。 -- [ ] 接入主循环的 Swipe/Wake 策略、一次性消费、会话失效与屏保计时。 -- [ ] 默认关闭；开发阶段先用 Mosaico 专属编译开关。交付前提供用户可操作的开启/关闭入口：设置页独立 `CAMERA` 开关，与显示设置共用保存/取消；已保存为打开时顶栏显示摄像头标识。 -- [ ] 无摄像头、采集超时、推理失败、内存不足时禁用视觉输入，保留触摸/按键/BLE，不以 fatal error 重启整个面板。初版不承诺热插拔自动恢复，可在重新启用时重试。 -- [ ] OTA 和配对期间暂停视觉工作，设置期间禁止动作；恢复后清空轨迹并重新确认手离开。处理进行中的 DMA/推理所有权后再停止。 +- 单手检测置信度至少 0.65；位置归一化到当前屏幕坐标。多手、越界、尺寸异常和不连续观测不构成有效挥手。 +- 挥手至少跨越三帧、持续 150–900 ms；主轴净位移至少 25%，至少为另一轴最大偏移的两倍，且不小于主轴累计路径的 75%。 +- 招手由三段交替水平运动构成，每段至少为画面宽度的 12%。一次连续运动只发出一个动作；再次接受挥动前需要约 400 ms 静止，帧间位移不超过 3% 视为静止。挥手后 1.5 秒内的反向挥动作为收手忽略。 +- OK 和点赞使用分类器的 `ok`、`like` 类别，置信度至少 0.85，稳定保持约 500 ms。相邻帧位移超过 5% 时跳过分类，优先保留轨迹采样频率。 +- 可接受的观测间隔上限为 400 ms。短暂漏检保留候选，长间隔或位置跳变重新开始跟踪。 +- 初始化、外部输入和 OK 触发后，挥手与姿态识别要求连续约 300 ms 无手，并距上次动作至少 500 ms。招手按交替轨迹单独判定。姿态停留不反复触发时钟切换。 -**完成条件：**下表的软件与实机验收通过；确认 HandDetect-only 能满足误触发目标，否则先评估 `five` 门控，再决定可用范围。无需等待 👌 OK 功能即可独立评审本阶段。 +相机安装校正先水平镜像再顺时针旋转 90°，随后应用屏幕旋转的逆变换。OV3640 的 640×480 图像缩为 320×240，SC101IOT 的 1280×720 图像缩为 320×180，旋转后将实际尺寸传给模型和预览。 -### P2：👌 OK 主动进入时钟 +## 资源与存储 -- [ ] 引入 HandGestureRecognizer；只对目标手分类，记录新增推理耗时与资源，重新检查 OTA 大小。 -- [ ] 稳定 `ok` 约 500 ms 作为初始停留确认值，增加简洁确认反馈；分数阈值由实测决定，不能把 top-1 输出直接视为可信命令。 -- [ ] 进入时钟后等待手离开，防止自然松手立即唤醒；保持 `ok` 只进入一次。`no_gesture` / `no_hand` 不映射到命令。 -- [ ] 若 P1 误触发需要 `five`，比较增加分类前后的误触、动作遗漏与延迟；不要求先静止半秒才能挥动。 +检测使用 ESPDet-Pico 224×224，分类使用 MobileNetV2 128×128。模型随代码内嵌在应用中，使用 7 MiB 双 OTA 槽，更新及回滚使用同一份代码和模型。组件版本与模型 SHA-256 见[依赖说明](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)。 -**完成条件：**正常页一次 👌 OK 进入、重复保持无副作用、自然松手不退出，随后独立挥手只恢复原页；开关关闭后不再采集。 +720p 双 UYVY 缓冲约占 3.52 MiB。启动预检查要求至少 9 MiB 连续空闲 PSRAM 和 96 KiB 内部内存，为模型及并行 UI 分配留出空间。ESP-DL 内部分配包含断言，预检查不构成运行时内存分配成功的保证。 -## 验证与决策门槛 +LVGL 使用 CLIB 分配器，避免独立固定大小的字形内存池。日志记录采集转换、检测、分类、反馈耗时以及内存和栈余量；`mean_cycle_ms` 不包含反馈和每轮固定 20 ms 等待，不能直接换算为帧率。 -以下数量和延迟是建议的首轮验收目标，P0 测量后再确认可实现范围，不是当前性能承诺。 +## 开发检查 -| 范围 | 检查与建议门槛 | -| --- | --- | -| 纯逻辑 | 在现有 Mosaico logic 测试入口增加少量有意义的轨迹用例：上下左右、抖动/斜向、漏检/目标跳变、持续停留、冷却/离开、旋转/上下文失效；不模拟整套相机驱动 | -| 屏保 | 若共享控制器有改动，扩展现有 panel_state 测试：首挥只唤醒、恢复原页、重复进入幂等、无效检测不延长超时；保留现有触摸/按键/远程用例 | -| 编译 | 手势开/关均构建；共享状态若变化，构建受影响目标。检查锁文件、镜像大小、分区和生成配置；测试镜像构建与测试执行分别记录 | -| 方向/识别 | 记录实际距离、光线与板方向；每朝向每方向 20 次，有效操作成功率目标 ≥90%，零反向，零单轮多次触发 | -| 唤醒/门控 | 每个用量页进入屏保再挥手，第一次只恢复；设置/配对/OTA 中持续挥手及恢复后均无遗留动作 | -| 误触 | 打字、拿杯、路人、静止手、照明变化等各类场景，合计至少 30 分钟无误触；不能将短期结果宣传为长期误触率 | -| 延迟/共存 | 从动作满足条件到 UI 响应 p95 目标 ≤300 ms；同时记录采集、转换、推理、主循环等待及动作总时长。与关闭功能时对比 UI 帧耗时和 BLE 更新延迟，恶化超过约 20% 时先优化/降频，不默认发布 | -| 资源/恢复 | 连续运行至少 30 分钟，堆低水位趋于稳定、无缓冲耗尽/看门狗；反复启停，缺摄像头与超时故障均可降级 | -| 功耗 | 同亮度、同 BLE 负载测关闭/用量页启用/屏保启用的电流；低频采集是否节能必须测量，S31/P4 的计算能力不能替代功耗证据 | - -固件构建前先运行 `eim --version` 和 `eim list`，按目标注册表使用 v6.1： +在仓库根目录使用目标指定的 SDK： ```sh +eim --version +eim list eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 eim run "idf.py -C firmware/targets/esp_mosaico/test_apps/logic build" v6.1 eim run "idf.py -C firmware/test_apps/panel_state build" v6.1 ``` -烧录、清除存储、发布另行按仓库授权规则执行。 - -## 当前实现与验证记录 - -上述阶段的复选框保留为端到端验收条件，未勾选不等于没有软件实现。已完成： - -- 固定 registry 依赖和模型包；左槽 OV3640 采集适配、双缓冲、UYVY 转 RGB888 和四向映射；来源见 [CAMERA_UPSTREAM.md](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)。 -- 单手轨迹、OK 500 ms 停留、挥手冷却与收手抑制、OK 后离开再触发、有界事件与上下文失效；主循环实现翻页、只唤醒、幂等进入时钟及受保护状态屏蔽。 -- 设置页独立 `CAMERA` 开关默认关闭，与显示设置共用保存/取消；顶栏摄像头标识区分采集中、暂停与失败，不显示文字提示；停用释放采集与模型，手动重试。ESP-DL 内部部分分配使用断言；资源预检查降低风险，但不保证所有运行时内存不足均能无重启降级。 -- 7 MiB 双 OTA 分区、Folder Push 上限和 Bridge 容量配置；无共享模型分区。 -- 手势开/关固件、Mosaico logic、共享 panel_state 测试镜像以及 AMOLED 2.16 固件构建通过。已完成手势/屏保主机 Unity 测试的 ASan/UBSan 检查及 Bridge 自动化测试。软件与硬件验证范围见[验证范围](../verification.md)，新增用例的执行情况以[手势验收记录](../validation/mosaico-gestures.md)为准；测试镜像尚未在设备运行。 - -下一步是按[首次烧录与实机验收](../validation/mosaico-gestures.md)安装并开启手势，检查摄像头方向、真实模型输出、UI 提示布局、推理耗时、误触、启停与功耗。未烧录、未发布，也未验证物理 A/B 回滚。不能把本次软件验证解释为 P0/P1/P2 的硬件验收通过。 - -## 参考来源 - -后续开发优先阅读[摄像头与手势仓库参考索引](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)：集中记录官方仓库、项目 fork、固定提交/组件版本、各仓库用途及本地代码入口。下列默认分支链接用于查阅，不代表本实现的锁定版本。 - -- [分享报告及交互修订](https://chatgpt.com/share/6ab29265-faa8-83e8-8e32-fa2bb8b8989b)；另一份为本次任务内粘贴的补充报告。 -- [ESP-DL Model Zoo](https://github.com/espressif/esp-dl/blob/master/models/README.md) -- [HandDetect CMake](https://github.com/espressif/esp-dl/blob/master/models/hand_detect/CMakeLists.txt) 与 [模型说明](https://github.com/espressif/esp-dl/blob/master/models/hand_detect/README.md) -- [HandGesture CMake](https://github.com/espressif/esp-dl/blob/master/models/hand_gesture_recognition/CMakeLists.txt)、[接口定义](https://github.com/espressif/esp-dl/blob/master/models/hand_gesture_recognition/hand_gesture_recognition.hpp) 与 [类别清单](https://github.com/espressif/esp-dl/blob/master/examples/hand_gesture_recognition/README.md) -- [ESP-VISION 支持板卡](https://docs.espressif.com/projects/esp-vision/en/latest/esp32p4/target-support/index.html) 与 [工程说明](https://github.com/espressif/esp-vision) -- [ESP-WHO](https://github.com/espressif/esp-who) -- [Mosaico 当前移植说明](../../firmware/targets/esp_mosaico/README.md)、[BSP 来源与本地修改](../../firmware/components/esp-mosaico-bsp/UPSTREAM.md)、[软件与硬件验收边界](../validation/software-acceptance.md) +Mosaico logic 包含轨迹、动作路由、图像坐标转换和设置持久化用例，共享 panel_state 覆盖屏保。构建测试镜像与运行用例是不同步骤，执行结果应对应具体源码和产物记录。仓库通用检查见[软件与发布验收](../validation/software-acceptance.md)。 diff --git a/docs/devices.md b/docs/devices.md index 5b330d4..3b2f597 100644 --- a/docs/devices.md +++ b/docs/devices.md @@ -8,6 +8,7 @@ | --- | --- | --- | --- | --- | | 屏幕 | 480×480 圆角方形 AMOLED | 135×240 LCD | 800×480 四灰阶墨水屏 | 480×480 AMOLED | | 翻页交互 | 底部触控标签与左右滑动 | 正面按键 | 三向拨轮 | 触控标签、左右滑动与 AI 键 | +| 摄像头手势（实验性） | — | — | — | 需左槽 CameraBoard，默认关闭 | | 用量页面 | 总览 / Codex / Claude | 总览 / Codex / Claude | 主页同屏显示 Codex 与 Claude，另有趋势页 | 总览 / Codex / Claude | | 断连保留最后用量 | ✅ | ✅ | ✅ | ✅ | | 电量与充电图标 | ✅ AXP2101 | ✅ M5PM1 | ✅ AXP2101 | ✅ BQ27220 | @@ -15,7 +16,7 @@ | 屏保用量细条 | ✅ | — | ✅ | ✅ | | 自动旋转 | ✅ 四向 | — 固定 0°/180°（menuconfig） | ✅ 四向 | ✅ 四向 | | 摇动 / 翻转唤醒 | ✅ | — | — | ✅ | -| 屏上设置页 | ✅ 亮度、自动时钟超时 | — 亮度仅 menuconfig | ✅ 刷新间隔、屏幕方向 | ✅ 亮度、自动时钟超时 | +| 屏上设置页 | ✅ 亮度、自动时钟超时 | — 亮度仅 menuconfig | ✅ 刷新间隔、屏幕方向 | ✅ 亮度、自动时钟超时、CAMERA 开关 | | 温湿度显示 | — | — | ✅ 时钟页 | — | | 用量趋势（48 点 / 30 分钟） | — | — | ✅ 需 FAT microSD 卡 | — | | 托盘/菜单栏左键切换时钟（`screen.toggle.v1`） | ✅ | — | ✅ | ✅ | @@ -32,6 +33,7 @@ RLCD 4.2 使用 400×300 黑白反射式 LCD，提供总览、Codex / Claude 详 ## 屏幕快捷操作 - **RLCD 4.2**：KEY 上一页，BOOT 下一页，长按 KEY 进入或退出设置；设置中 KEY 选择项目、BOOT 修改选项。KEY 可关闭当前用量告警。OTA 待确认时 KEY 确认，长按 KEY 拒绝。 -- **480×480 触屏设置（微雪 AMOLED 2.16 与 ESP-Mosaico）**：长按触屏约 0.8 秒进入，两块板共用同一套设置。亮度即时预览；AUTO CLOCK 可选择 OFF、1、5、10、30 分钟。SAVE 保存并在重启后保留，CANCEL 撤销未保存调整。OFF 仅关闭自动进入时钟，托盘/菜单栏左键仍可主动切换；微雪另有 PWR 侧键，ESP-Mosaico 另有 AI 键。 +- **480×480 触屏设置（微雪 AMOLED 2.16 与 ESP-Mosaico）**：长按触屏约 0.8 秒进入，两块板均支持亮度与自动时钟设置。亮度即时预览；AUTO CLOCK 可选择 OFF、1、5、10、30 分钟。SAVE 保存并在重启后保留，CANCEL 撤销未保存调整。OFF 仅关闭自动进入时钟，托盘/菜单栏左键仍可主动切换；微雪另有 PWR 侧键，ESP-Mosaico 另有 AI 键。 +- **ESP-Mosaico 实验性手势**：安装左槽 CameraBoard 后，在设置中打开 CAMERA 并保存。用量页向任意方向挥手或左右招手切到下一页；OK 或点赞稳定保持约半秒进入时钟，挥手或招手唤醒。点击顶栏摄像头标识切换预览。完整说明见[手势指南](validation/mosaico-gestures.md)。 - **Windows 托盘与 macOS 菜单栏**：左键切换支持设备的时钟；鼠标悬停在 Bridge 图标上滚动，上滚查看上一页，下滚查看下一页，页面集合与顺序由各设备决定。操作作用于所有已连接且支持该命令的设备；设备处于时钟时会先回到额度页面。快速连续滚动会限速，OTA 期间不执行翻页。 - 滚轮翻页要求设备通告 `screen.page.v1`；未通告的设备只接收其支持的命令。 diff --git a/docs/roadmap.md b/docs/roadmap.md index abb0cbc..f6c9be7 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -19,7 +19,7 @@ | [AMOLED 性能参考](validation/amoled-performance.md) | 内存、刷新耗时与测量条件 | | [ePaper 刷新参考](validation/epaper-refresh.md) | 局刷数据及光学验证边界 | | [接入架构](design/open-device-access.md) | 状态归属、连接与更新生命周期 | -| [Mosaico 手势输入规划](design/mosaico-gesture-input.md) | 摄像头、挥手翻页/唤醒与分阶段验收 | +| [Mosaico 实验性手势](design/mosaico-gesture-input.md) | 操作规则、处理链路与识别参数 | | [接入协议](../protocol/open-device-access.md) | 设备线上字段与校验约束 | | [硬件移植](PORTING.md) | 新目标目录与硬件验收 | | [设计语言](design/amoled-ui.md) | AMOLED 界面规范 | diff --git a/docs/validation/mosaico-boot.md b/docs/validation/mosaico-boot.md index d680c7f..74c6954 100644 --- a/docs/validation/mosaico-boot.md +++ b/docs/validation/mosaico-boot.md @@ -1,76 +1,24 @@ -# Mosaico 开机耗时 +# Mosaico 启动流程 -用户反馈开机后白屏闪一下，约 10 秒才出现连接、数据和旋转响应。以下时间取自 UART0 启动日志的时间戳（上电起算，单位 ms），调整前后各一次冷启动，均为手势固件、相机开启、Bridge 已配对并在运行。单次测量不代表分布；Bridge 的连接时刻取决于主机端扫描。 +## 显示与连接 -## 调整前（2026-10-02） +屏幕初始化先建立共享 I2C 总线和供电。CO5300 亮度寄存器以零启动，`bsp_display_new()` 设置方向后发出 Display On；首次刷新完成后，界面通过 120 ms 亮度过渡显示画面，避免显示未初始化的帧存。 -| 阶段 | 结束时刻 | 耗时 | -|---|---|---| -| 引导程序加载并校验应用（约 5.4 MB） | 989 | 约 960 | -| PSRAM 初始化、代码与只读数据拷入 PSRAM、自检 | 1940 | 约 950，其中自检约 190 | -| 屏幕与触摸初始化 | 3478 | 约 1540 | -| BMI270 | 3885 | 约 410 | -| BQ27220 | 8276 | 约 4390 | -| 按键、BLE 初始化，开始广播 | 8496 | 约 220 | -| 主循环开始 | 8502 | | -| Bridge 连接 | 11008 | | -| 加密完成 | 11607 | | -| 收到对时 | 12094 | | +屏幕就绪后创建电量计初始化任务，随后启动 BLE 广播，再初始化 BMI270 和按键。外设初始化期间收到的 BLE 事件进入队列，由主循环处理；Bridge 的实际连接时刻还取决于主机扫描。 -- BQ27220：驱动的 `bq27220_seal()` 无条件等待 `CONFIG_BQ27220_SEAL_SETTLE_MS`（2000 ms）。`bq27220_create()` 在校验电池配置后已封存一次，`bsp_battery_init()` 随后再次调用，合计约 4 秒。 -- 白屏：CO5300 初始化序列把亮度寄存器 0x51 设为 0xFF 并发出 Display On，随后等待 600 ms；此时尚未写入任何一帧，面板显示上电后的帧存内容。 -- 主循环在全部外设初始化之后才开始，旋转、渲染和 BLE 事件处理都排在其后。 +## 电量计 -## 调整 +BQ27220 在独立任务中核对配置，必要时写入配置，再封存并确认状态。`bsp_battery_init()` 复用驱动已完成的封存，不重复执行。任务通过原子就绪标志将电量计交给主循环读取；就绪前不显示有效电量，OTA 电源判定保持未就绪。 -- 电量计在独立任务中初始化，就绪前电池图标隐藏，OTA 电源判定保持未就绪。封存等待本身未改动。 -- CO5300 以零亮度启动，初始化序列不再包含 Display On 及其 600 ms 等待（`bsp_display_new()` 在设置方向后发出 Display On）；界面在首次刷新完成后经现有的亮度过渡淡入。板级改动记录在 `firmware/components/esp-mosaico-bsp/UPSTREAM.md`。 -- BLE 在屏幕初始化后立即启动，其余外设初始化期间即可被连接；链路事件在队列中等待主循环。 +驱动的 `CONFIG_BQ27220_SEAL_SETTLE_MS` 默认值为 2000 ms，Mosaico 配置为 200 ms。等待结束后驱动仍读取状态确认封存；确认失败时初始化失败，不发布有效电量。该值是启动流程的实验性配置，不是芯片时序保证。 -## 调整后（2026-10-02） +电池图形创建时按无数据状态着色，主循环获得读数后更新，避免第一帧被误认为满电。电量计配置重写会恢复配置中的默认容量，不能将此过程视为保留电量计学习结果。 -| 里程碑 | 调整前 | 调整后 | -|---|---|---| -| 屏幕初始化完成 | 3382 | 2783 | -| 开始广播 | 8496 | 3196 | -| 主循环开始 | 8502 | 3778 | -| Bridge 连接 | 11008 | 4005 | -| 加密完成 | 11607 | 4574 | -| 收到对时 | 12094 | 5059 | -| BQ27220 就绪 | 8276 | 7901 | +## 维护入口 -应用通过 ESP-IDF v6.1 构建，按六段写入 ota_0 并全部通过设备哈希校验，未写入 NVS、分区表、引导程序或 OTA 数据。日志中没有新增的错误或告警。白屏是否消除、淡入是否正常、电池图标延后出现的观感，以及电量计与 IMU、触摸并发访问 I2C 的长期稳定性，尚待实机确认。 +- [app_main.cpp](../../firmware/targets/esp_mosaico/main/app_main.cpp)：初始化顺序、后台电量计任务和主循环轮询。 +- [display_ui.cpp](../../firmware/targets/esp_mosaico/main/display_ui.cpp)：首次刷新与亮度过渡。 +- [sdkconfig.defaults](../../firmware/targets/esp_mosaico/sdkconfig.defaults)：封存等待及目标配置。 +- [BSP 本地修改](../../firmware/components/esp-mosaico-bsp/UPSTREAM.md)：显示启动序列和电量计集成约束。 -## 去掉重复封存（2026-10-02） - -`bsp_battery_init()` 不再在 `bq27220_create()` 之后再次封存；驱动在返回句柄的两条路径上都已封存并确认。应用通过 ESP-IDF v6.1 构建，按六段写入 ota_0 并全部通过设备哈希校验。 - -| 冷启动 | 配置核对 | BQ27220 就绪 | -|---|---|---| -| 烧录后第一次 | 首项 `gauging_config`（0x929B）预期 3345、读到 4138，驱动重写全部配置并重新初始化 | 10217 | -| 第二次 | 核对通过，跳过更新 | 5890 | - -第一次启动中用户观察到电池显示先满、再空、后恢复；日志未打印电量值，推断为电量计重新初始化期间的读数。重写包含学习容量，电量计学到的满充容量已回到默认 80 mAh。 - -随后一次烧录后的第一次启动再次出现相同的不一致（同一字段、同样读到 4138）并重写，就绪时刻 10220；其后的普通重启核对通过，就绪时刻 5888。已记录的启动中，两次不一致都发生在经过 ROM 下载模式烧录后的第一次启动，普通重启均核对通过，包括紧接在一次核对通过的后台初始化启动之后的重启。这不支持“后台初始化导致下一次启动读错”，当时看来与“设备停留在下载模式期间电量计配置恢复为其他值”一致；机制未确认，后续记录见“电池显示提前”一节。更早一次记为“烧录后核对通过”的启动之前，设备已自行离开下载模式，可能存在未记录的启动。驱动读取配置时不校验校验和。每次重写都会把学习到的满充容量恢复为默认 80 mAh。 - -## 未处理 - -- 屏幕初始化中 Sleep Out 之后的 600 ms 等待；`esp_lcd_co5300` 的默认序列为 60 ms，未对照面板模组的上电时序规格。 -- 应用启动前约 1.9 秒：引导程序校验应用、代码与只读数据拷入 PSRAM、PSRAM 自检（约 190 ms）。 -- BMI270 初始化约 400 ms。 -- Bridge 发现并连接设备的耗时在主机端。 - -## 电池显示提前（2026-10-02） - -用户反馈电池显示出现得慢；其他启动项不再调整。 - -- 电池格在创建时即按“无数据”着色。此前未设置颜色，第一帧显示为界面库默认的浅色，看起来像满格，随后依次变为空、真实电量。 -- `CONFIG_BQ27220_SEAL_SETTLE_MS` 由驱动默认的 2000 改为 200。驱动在等待后仍读取状态确认已封存，未封存时初始化失败、电池隐藏。200 ms 没有数据手册依据，属试验值。 -- 电量计任务在屏幕初始化后立即启动，先于 BLE 和 BMI270 初始化。 - -应用通过 ESP-IDF v6.1 构建，按六段写入 ota_0 并全部通过设备哈希校验。烧录结束后设备自行离开下载模式并启动，该次启动未被记录。 - -其后记录到一次普通重启：电量计初始化在 3004 ms 开始，核对再次在同一字段读到 4138，驱动重写配置，就绪时刻 7595；200 ms 等待后的封存确认通过。这次启动不是烧录后的第一次启动，此前应用已连续运行至少四分钟，因此上文“不一致只出现在经过下载模式之后”的推断不成立。随后两次快速重启的 UART 记录大量丢失，没有采到电量计日志，无法判断。 - -未确认事项：200 ms 等待是否会使电量计在下一次启动时配置异常；配置不一致与关机时长的关系；普通启动中电量计的实际就绪时刻。若确认与 200 ms 有关，恢复 `CONFIG_BQ27220_SEAL_SETTLE_MS` 的默认值即可，电量计任务提前启动可以保留。 +启动耗时取决于电量计是否重写配置、外设状态和 Bridge 连接时机。比较启动日志时应使用相同硬件、配置和供电条件，并分别记录显示就绪、广播、主循环和数据到达时刻。 diff --git a/docs/validation/mosaico-gestures.md b/docs/validation/mosaico-gestures.md index d7cac8d..dfae811 100644 --- a/docs/validation/mosaico-gestures.md +++ b/docs/validation/mosaico-gestures.md @@ -1,162 +1,51 @@ -# Mosaico 手势固件：安装与验收 +# Mosaico 实验性手势：使用与安装 -本功能在设备本地运行：任意方向挥手或像打招呼那样左右招手切到下一页，末页后回到首页；时钟中首次挥手或招手只恢复原页面；👌 OK 或 👍 点赞稳定停留约 500 ms 进入时钟。一段连续动作只触发一次，手停住约 0.4 秒或离开画面后才接受下一次；挥手后约 1.5 秒内的反向挥动视为收手而忽略。进入时钟后需要手离开画面再操作。触摸和 AI 键继续可用。 +本地摄像头手势是**实验性功能**，默认关闭。需要左槽 CameraBoard，支持 OV3640 和 SC101IOT，固件自动识别传感器。图像只在设备内处理，不上传、不保存。识别效果受光线、距离和手部是否完整入镜影响。 -这是供实机验收的实现。软件测试与固件构建不能证明相机成像、方向、识别率、功耗或长期稳定性；这些仍需在目标设备验证。 +## 开启与操作 -## 当前分区与构建 +安装相机板前先断电。开机后长按触屏进入设置，打开 `CAMERA` 并点击 `SAVE`。关闭该开关并保存即可停止采集；`CANCEL` 撤销未保存的调整。 -16 MiB Flash 使用两个 7 MiB 应用槽，模型与代码共同更新。NVS 位于 `0x9000`，ota_0 位于 `0x10000`，otadata 位于 `0x810000`，ota_1 位于 `0x820000`。Folder Push 限额为 7 MiB + 512 字节，目标注册表和 Bridge 下载上限同步匹配。 +| 页面 | 动作 | 效果 | +| --- | --- | --- | +| 用量页 | 向任意方向挥手，或左右招手 | 切到下一页，按总览 → Codex → Claude 循环 | +| 用量页 | 👌 OK 或 👍 点赞稳定保持约半秒 | 进入时钟 | +| 时钟 | 挥手或左右招手 | 恢复进入时钟前的页面 | -在仓库根目录运行： - -```sh -eim --version -eim list -eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 -``` - -确认日志含 `Project build complete`，并检查生成配置启用 CLIB 分配器、OV3640 与 SC101IOT。SC101IOT 使用完整 1280×720 UYVY，驱动标称 15 fps；OV3640 使用 640×480 UYVY。 - -## 有线烧录注意事项 - -相机板安装在左槽，安装前断电。在已授权的有线烧录中，按住 BOOT 再开机进入 ROM 下载模式。Mosaico 实测整段擦写大应用可能断连；不要直接使用 `idf.py flash` 或将未拆分的大应用交给 `@flash_args`。 - -将应用按 1 MiB 拆分，地址从目标应用槽起点逐段递增，使用 ESP-IDF v6.1 环境中的 esptool，以 `--chip esp32s31 --no-stub`、460800 波特率、DIO/80 MHz/16 MB 参数写入，并指定 `--after no-reset`。烧录前校验分段拼接内容与构建应用一致，烧录后确认每段设备哈希校验成功，再通过 POWER 关机重启。 - -仅更新当前应用时，不写入 NVS、分区表或初始 OTA 元数据。需要完整安装时，使用构建产物匹配的 bootloader、分区表及初始化数据；应用仍分段写入。不要通过整片擦除或包含填充区域的合并镜像声称保留配对与设置。 - -## 启用与操作 - -1. 长按现有显示设置入口，打开 `CAMERA` 开关并点击 `SAVE`。默认关闭；`CANCEL` 不会启用摄像头。 -2. 顶栏摄像头标识变为亮色后，先把手移出画面至少约半秒，再开始动作。 -3. 正常页朝任意方向挥动或左右招手均为下一页，一段连续动作只翻一页；再翻前先停住或移开手。屏保中第一次挥动只恢复原页。 -4. 正常页做 👌 OK 或 👍 点赞并短暂停留；进度条完成才进入时钟。进入后先移开手，下一轮挥手再唤醒。 -5. 设置、配对确认和 OTA 期间不接受手势，标识转暗。关闭 `CAMERA` 开关并保存后释放采集和模型资源，顶栏标识消失。 - -顶栏标识带斜杠表示采集失败：相机未能启动或连续采集失败、资源预检查未通过，或驱动资源未能安全释放。检查左槽相机后，把 `CAMERA` 关闭并保存、再打开并保存可重试；仍带斜杠时重启设备。具体原因见 UART0 日志。 - -摄像头占用左侧排针上的 USB Serial/JTAG（GPIO33/34）。Type-C 口是 USB OTG，只在 ROM 下载模式下枚举，与摄像头无关；应用日志只从 UART0 输出（右侧排针 H1 的 15 脚 TX0 / GPIO58，或底板背面调试焊盘 TX，115200）。采集失败时每 5 秒打印一次 `capture unavailable: <步骤> (code N)`。板上没有 RST 键：复位用 POWER 键关机再开机，下载模式为关机后按住 BOOT 再开机。日志只报告动作、周期时间、内存和栈余量，不输出图像或用户数据。屏保时摄像头仍工作，因此本功能不是深睡眠唤醒，也未验证续航收益。 - -## 实机验收顺序 - -- 首先确认原有触摸、AI 键、BLE、显示旋转和 OTA 确认界面工作正常。 -- 确认 `CAMERA` 开关保存后重启仍保持，切换开关会启用 `SAVE`、`CANCEL` 会恢复；保存为打开时顶栏标识出现，采集中为亮色，设置页、配对或 OTA 期间为暗色，失败时带斜杠；关闭并保存后标识消失。 -- 检查相机能连续采集，记录 `gesture_input` 周期与内存日志；缺相机时面板应继续工作。 -- 在每个屏幕朝向检查左右语义。默认先水平镜像再顺时针旋转 90°，与官方参考实现的逆时针 90° 正立画面一致；若方向或手形识别不正确，使用 `menuconfig → Camera gestures` 校准后重建，不能把主机坐标转换测试当作硬件方向验证。 -- 分别在三个用量页进入时钟，再挥手，确认回到原页且不继续翻页。保持 OK、自然松手和单帧漏检都不应反复切换。 -- 测试设置/配对/OTA 期间持续挥手，退出后不能回放旧动作；随后移开手再做新动作应恢复。 -- 记录实际距离、照明和方向；每朝向每方向至少 20 次，并在打字、拿杯等场景观察至少 30 分钟误触。连续运行及反复启停，检查内存是否稳定。 -- 同亮度、同 BLE 负载比较手势关闭、正常页启用、屏保启用时的电流和界面响应。尚未测得的数据不得标为通过。 - -识别距离、方向标定、模型延迟、功耗、实际摄像头热插拔和 A/B 回滚效果均为硬件验收项；初版只提供关闭/开启重试，不承诺热插拔自动恢复。 - -## 初始实现的软件验证记录 - -- ESP-IDF v6.1：手势开启/关闭固件均构建成功；Mosaico logic 和共享 panel_state 测试镜像构建成功；受共享屏保接口影响的 AMOLED 2.16 固件构建成功。 -- 手势轨迹与屏保 Unity 测试曾在主机实际运行并通过 ASan/UBSan。之后新增的漏检、低置信度、慢帧与位置跳变用例仅随 logic 测试镜像编译，尚未在主机或 MCU 上运行；测试镜像未在 MCU 上运行。 -- `CAMERA` 开关与顶栏标识只经过固件构建，布局与触控需实机确认。 -- Bridge 全部测试通过，包含新 7 MiB 发布清单容量和第三方许可检查。 -- 初始验证时未进行实机烧录；后续烧录与设备验证见下文。ESP-DL 内部模型分配包含断言，预检查不能替代内存压力与启停实测。 - -## 时钟黑屏排查（2026-09-23） - -故障固件使用 LVGL 内置 64 KiB 内存池，未启用扩容。原始 UART 日志中的 ELF 哈希与排查用 ELF 匹配；进入时钟后，`swdraw` 的看门狗现场持续停在 `lv_draw_unit_draw_letter` 的 `LV_ASSERT_MALLOC(draw_buf)`。反汇编确认该地址为自跳转指令，前置分支仅在字形缓冲分配返回 NULL 时进入。因此，已确认直接故障为字形缓冲分配失败后断言死循环；现有日志不能区分内存池总余量不足与碎片化，也没有 main 分支的运行时内存池余量对照。 - -Mosaico 默认配置改为 `CONFIG_LV_USE_CLIB_MALLOC=y`，使界面对象和字形缓冲使用 ESP-IDF 标准堆。已有工作目录还需在生成的 `sdkconfig` 中关闭 `CONFIG_LV_USE_BUILTIN_MALLOC` 并启用 CLIB，重新构建后检查 `build/config/sdkconfig.h`。标准堆是否使用 PSRAM 由 ESP-IDF 的分配策略决定，不能将 CLIB 等同于强制 PSRAM 分配。 - -本轮 ESP-IDF v6.1 构建完成；生成配置启用 CLIB 和调试预览，最终 ELF 的 lv_malloc_core 直接跳转到标准 malloc。通过 UART0 被动读取的修复前设备日志仍停在同一断言地址。修复镜像已按 1 MiB 分段写入应用区，六段均通过设备哈希校验；未写入 NVS、分区表或 OTA 元数据。重启日志的 ELF 哈希与修复产物一致，摄像头和 BLE 均恢复运行。用户确认通过 OK 进入时钟后显示正常；后续 UART 采集日志持续输出，未再出现看门狗报警。日志另捕获到 SwipeLeft/Next 与 SwipeRight/Previous 各一次；这只能证明两个方向均已触发，不能代替触发率验收。 - -后续压力验收：保持调试预览开启，反复通过 OK 进入时钟并唤醒，覆盖分钟切换、触摸、BLE 和摄像头启停；确认画面正常、无 `swdraw` 看门狗，且现有 `internal_free`、`psram_free` 日志无持续下降。CLIB 后端不提供有效的 `lv_mem_monitor` 统计，应使用 ESP-IDF 堆统计。左右挥手的触发率仍待单独验证。 - -## 近距离取景调整 +一段连续挥动只执行一次，手停稳后再做下一次动作。触摸、AI 键和 Bridge 操作继续可用；设置、配对确认和 OTA 期间暂停手势输入。 -SC101IOT 使用完整 1280×720 画面，避免 VGA 模式的中心裁切；等比例缩为 320×180 RGB888，旋转后将实际尺寸传给模型和预览。OV3640 保持 640×480 采集、320×240 RGB888。手势阈值保持不变，以便单独验证取景范围变化。 +## 摄像头标识与预览 -本版应用与 logic 测试镜像均已通过 ESP-IDF v6.1 构建，新增图像比例、旋转、镜像和边缘保留用例仅完成编译，尚未运行。应用已按六段写入设备，全部通过设备哈希校验；未写入 NVS、分区表或 OTA 元数据。重启日志确认运行镜像与产物匹配，并成功采集 1280×720 UYVY；观察窗口内无看门狗或采集失败，记录到一次右挥翻页。稳定采样的平均处理周期约 105～123 ms，PSRAM 空闲约 4.39 MB。用户反馈近距离取景有所改善，但旋转后的竖幅横向空间不足；具体距离、OK 识别及左右挥手触发率尚未验收。检查启动日志包含 `camera UYVY 1280x720`，预览应包含更完整的场景且手形无拉伸；分别在约 40、30、25 cm 检查完整入镜、OK 进入时钟与左右挥手。记录实际识别率、平均周期、帧超时和内存余量，确认反复关闭/开启 CAMERA 与时钟切换正常。25～30 cm 是测试目标，不是已验证的识别距离。 +打开 `CAMERA` 后，顶栏显示摄像头标识：采集中为亮色，暂停时变暗，失败时带斜杠。关闭并保存后标识消失。 -## 空中挥手的页面方向 +在普通用量页点击摄像头标识可显示或隐藏预览。预览保留画面比例，显示图像和手部定位框，可用于调整摆放位置。预览默认隐藏，重启后恢复隐藏；关闭预览不影响识别。时钟、设置、配对和 OTA 界面隐藏预览，返回普通页面后恢复本次选择。 -空中手势按导航项顺序选择页面：右挥或下挥为下一页，左挥或上挥为上一页，末尾按现有页面顺序循环。触摸滑动语义保持不变；时钟中首次挥手仍只唤醒。前述日志中的 SwipeRight/Previous 等记录属于调整前版本，不作为当前映射的验收证据。 +标识带斜杠时，断电检查左槽相机连接，再开机重试。也可关闭 `CAMERA` 并保存，再打开并保存；仍无法启动时重启设备。屏保期间摄像头继续工作，开启手势会增加耗电。 -方向调整后的应用与 logic 测试镜像均通过 ESP-IDF v6.1 构建；路由测试尚未运行，方向修正已随四向手势应用烧录，方向一致性待实机确认。 +## 构建 -## 上下挥手 - -上下挥手与左右挥手共用单手置信度、150～900 ms 动作窗口、至少三帧、25% 主轴净位移和重复触发抑制。斜向轨迹及往返抖动不应翻页；时钟中任一方向首次挥动只恢复原页。实机需分别验证上挥上一页、下挥下一页，以及一次动作不会重复翻页；新增四向逻辑尚未实机验收。 - -四向手势应用和 logic 测试镜像均通过 ESP-IDF v6.1 构建；扩展后的轨迹、斜向/抖动拒绝及动作路由测试尚未运行。应用已按六段写入设备，全部通过设备哈希校验；仅更新应用分区，未写入 NVS、分区表或 OTA 元数据。烧录结束后设备停留在下载模式，后续日志及用户反馈确认：手离开镜头后，上挥可唤醒时钟；其余方向和连续操作的完整验收仍待完成。 - -## 相机图标预览开关 - -预览默认隐藏，点击顶部相机图标切换显示；隐藏时跳过预览绘制，手势识别继续运行。应用通过 ESP-IDF v6.1 构建，已按六段烧录并全部通过设备哈希校验。仅更新应用分区，重启后已采集到分阶段性能日志，图标点击行为仍待用户确认。本版本包含采集转换、检测、分类和预览反馈的分阶段耗时统计。 - -预览交互验收需检查：启动隐藏、点击图标开关、图像比例及手部定位框、无调试文字；进入时钟或受保护页面后隐藏，恢复识别并返回普通页面后恢复本次选择。隐藏期间应跳过预览显示锁和绘制；实际锁竞争与帧率变化仍需测量，不能以构建结果代替性能验收。 - -## 性能基线与后续优化（2026-09-23） - -预览关闭且未执行 OK 分类的三个连续约 10 秒窗口中，每窗口处理 80 帧，约 8 fps。平均采集与转换耗时 18.6 ms，手掌检测（含前后处理）86.8 ms，反馈约 0.006 ms，循环另有固定 20 ms 等待，总帧间隔约 125 ms。`mean_cycle_ms` 不含反馈及固定等待，不能直接取倒数作为识别帧率。该窗口不代表正常页检测到手后执行 OK 分类的完整负载。 - -[官方手掌检测基准](https://github.com/espressif/esp-dl/tree/master/models/hand_detect)给出同模型 S3 合计 132.8 ms、P4 合计 50.9 ms，未列 S31。[官方 S31 案例](https://esp32-s31.espressif.com/en/official-projects/6)中同系列 ESPDet-Pico 猫检测合计 94.9 ms，仅用于量级参考，不能视为相同模型的直接对比。当前 CPU 为 320 MHz，PSRAM 为 250 MHz。 - -后续按以下顺序评估，尚未实现，收益不得作为已验证结果： - -1. 缩短固定 20 ms 等待，并检查 BLE、显示、看门狗和任务公平性。其他耗时不变时，理论上可接近 9.5 fps。 -2. 将采集等待与图像转换继续拆分计时，再对照官方相机实现评估硬件缩放、旋转及转换。保留完整传感器视野，不通过中心裁切换取速度。 -3. 测量正常页有手时的 OK 分类成本；评估移动时优先跟踪、稳定后再分类。时钟中已跳过 OK 分类，不能重复计算这项收益。 -4. 若上述收益不足，再评估更轻模型或检测与轻量跟踪交替运行；验证快速运动、遮挡、多手及重新捕获，避免累计漂移。 -5. 保留至少三帧确认，结合实际帧率评估 150 ms 最短动作时间、位移及位置跳变阈值。分别测量左右与上下挥手的成功率、误触率及唤醒延迟。 - -近距离优化需区分光学视野与识别条件：当前已使用 SC101IOT 完整画面；软件缩小预览不能扩大视角。可评估较小挥手幅度及大手掌框的过滤阈值，部分手掌入镜的识别可靠性需实测。预览开关的性能对比应采用相同手势、页面与照明条件。 - -## 连续挥手与帧间隔（2026-10-01） - -用户反馈：开启预览时 OK 无法进入时钟；每次动作后要求手离开画面使连续操作迟钝。 - -- 帧间隔上限由 250 ms 调整为 400 ms。原上限接近实际帧间隔，OK 分类与预览使间隔变长后，每一帧都会重置跟踪和 OK 停留计时。该原因由代码和上文帧间隔记录推断，未取得开启预览时的分阶段耗时日志。 -- 动作事件在预览绘制之前入队，预览等待显示锁不再占用事件 300 ms 的有效期。 -- 挥手后不再要求手离开：500 ms 冷却内的样本属于同一次挥动，不重复触发；1.5 秒内的反向挥动视为收手并忽略，同向和垂直方向可再次触发。挥手引起的页面或屏保变化不要求重新确认手离开。 -- OK 进入时钟、多手、触摸、AI 键、连接变化以及设置/配对/OTA 恢复后，仍要求连续约 300 ms 无手。 -- 帧处理期间上下文变化时，该帧不再作为输入，但仍绘制预览；此前这些帧既不识别也不显示。挥手后预览停顿是否由此引起尚未在实机确认，页面重绘对显示锁和 CPU 的占用未测量。 - -已知取舍：挥手后 1.5 秒内有意做的反向挥手会被忽略；收手晚于 1.5 秒会被识别为反向翻页。两个时间均为试验起点，需按实机误触和漏触发调整。 - -手势用例通过临时 Unity 兼容头在主机编译运行，9 条全部通过 ASan/UBSan；新增的 300 ms 帧间隔 OK 用例在 250 ms 上限下失败。应用与 logic 测试镜像（esp32s31）通过 ESP-IDF v6.1 构建。测试镜像未在 MCU 上运行；预览下的 OK、连续挥手、收手抑制和误触率均待实机验收。 - -## 生命周期与输入失效修复（2026-10-02） - -- 分类器在采集启动时加载模型，确保尚未执行分类就进入设置、配对或 OTA 时也能安全释放。单手分类直接使用 `HandGestureCls::run_crop`；锁定版本的懒加载包装器不保证未加载模型时的析构安全。 -- 应用显式标记已接受手势造成的上下文变化，保留该手势的冷却与释放状态。触摸、AI 键、远程操作及其他上下文变化均重新等待手离开，不再通过距上次挥手的时间推断变化来源。 - -10 条手势与图像用例通过临时 Unity 兼容头在主机运行，ASan/UBSan 无错误；包含冷却期间外部输入阻断后续动作、手离开后恢复识别的回归用例。分类器使用锁定依赖的构造和析构源码、替代推理接口，在填充非零内存的主机分配器下连续创建并释放 20 次，ASan/UBSan 无错误。应用及 logic 测试镜像通过 ESP-IDF v6.1 构建。修复应用已按六段写入设备，并写入匹配的引导程序、7 MiB 双槽分区表和初始 OTA 数据，九项写入均通过设备哈希校验。烧录前后 NVS 区域校验值一致，未执行全片擦除或自动复位。设备保留在下载模式；重启后的相机启停、外部输入后重新识别及连续挥手仍需实机确认。 - -## 单向轮换与点赞进入时钟（2026-10-02） - -- 四个方向的挥手均为下一页，按导航项顺序循环；前文“空中挥手的页面方向”和“上下挥手”中的上一页映射不再适用。触摸滑动仍可双向翻页。 -- 1.5 秒内的反向挥动仍视为收手。所有方向效果相同，因此该规则不再丢弃有意的返回操作；返回上一页需再挥两次，每次间隔不少于 500 ms 冷却。 -- 分类结果为 `ok` 或 `like` 时均计入进入时钟的停留计时，阈值仍为 0.85、停留约 500 ms。分类器只返回置信度最高的一类。 +在仓库根目录运行： -手势与图像用例通过临时 Unity 兼容头在主机运行，10 条全部通过 ASan/UBSan，其中路由用例已改为四个方向均为下一页。应用通过 ESP-IDF v6.1 构建，生成配置启用手势输入；logic 测试镜像未重新构建。烧录前读取 OTA 数据确认设备从 ota_0 启动；应用已按六段写入 ota_0，全部通过设备哈希校验，未写入 NVS、分区表、引导程序或 OTA 数据，未自动复位。设备保留在下载模式，重启后的行为尚未确认。点赞分类只经过固件构建，没有主机用例覆盖。握杯、握拳等接近点赞的手形是否误入时钟，以及点赞的识别率和距离，均待实机验收。 +```sh +eim --version +eim list +eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 +``` -## 招手识别与挥动采样（2026-10-02） +构建产物为 `firmware/targets/esp_mosaico/build/mosaico_usage_panel.bin`。生成配置应启用 CLIB 分配器、OV3640 和 SC101IOT；摄像头选项位于 `menuconfig → Camera gestures`。 -用户反馈单向轮换固件仍不灵，期望的动作是像打招呼那样左右招手。此前的判定要求单向净位移达到 25%，并要求净位移不小于路径的 75%，来回摆动因此被拒绝。 +16 MiB Flash 使用两个 7 MiB 应用槽，模型与代码共同更新。NVS 位于 `0x9000`，ota_0 位于 `0x10000`，otadata 位于 `0x810000`，ota_1 位于 `0x820000`。Folder Push 限额为 7 MiB + 512 字节。 -- 新增招手：水平方向连续三段交替移动、每段不少于画面宽度的 12%，识别为一次下一页；幅度达到 25% 的第一段仍按挥手立即触发。 -- 一段连续动作只触发一次：挥手或招手触发后，需手静止约 400 ms 或离开画面才接受下一次，取代原 500 ms 冷却。低置信度的手视为仍在运动。帧间位移不超过 3% 视为静止。 -- 行程起点跟随静止的手，动作窗口从手开始移动时起算。此前起点固定在手首次出现时，停留后再挥动可能跨过 900 ms 窗口而失效。 -- 相邻两帧位移超过 5% 时跳过手形分类，避免挥动期间帧间隔被分类拉长；手形分类因此比手停稳晚一帧开始。分类耗时尚未在设备上测量，帧率收益未验证。 -- 有手时逐帧输出一行观测日志（千分比坐标及送入跟踪器的时间），可用于在主机回放判定；原为每秒一行。 +## 有线烧录 -以上阈值均为试验起点，没有设备数据支持。已知风险：静止手的检测抖动若超过 3%，触发一次后只有移开手才能再次触发；鼠标或键盘操作中的手在画面内来回移动可能被识别为招手；招手频率接近帧率的一半时采样可能看不到完整摆幅。 +按住 BOOT 再开机进入 ROM 下载模式。整段擦写大应用可能断连，应用按 1 MiB 分段烧录，不直接使用 `idf.py flash` 或未拆分的 `@flash_args`。 -13 条手势与图像用例通过临时 Unity 兼容头在主机运行，ASan/UBSan 无错误；新增招手、停留后挥动和分类门控用例，并确认停用招手规则或起点跟随后对应用例失败。原“挥手后无需停顿即可同向重复”的用例已按新规则改写。应用通过 ESP-IDF v6.1 构建；logic 测试镜像未重新构建。烧录前读取 OTA 数据确认设备从 ota_0 启动。第一次写入在第一段约 37% 处 USB 断连，应用区留下不完整镜像，原因未确认；重新进入下载模式后从第一段起重写，六段全部通过设备哈希校验。仅写入 ota_0，未写入 NVS、分区表、引导程序或 OTA 数据，未自动复位；设备保留在下载模式，重启后的行为尚未确认。招手识别率、误触率、连续翻页节奏及分类门控对 OK 和点赞的影响均待实机验收。 +地址从目标应用槽起点逐段递增。使用 ESP-IDF v6.1 环境中的 esptool，参数为 `--chip esp32s31 --no-stub`、460800 波特率、DIO/80 MHz/16 MB，并指定 `--after no-reset`。烧录前核对分段拼接内容与构建应用一致，烧录后确认每段设备哈希校验成功，再通过 POWER 关机重启。 -## 时钟中招手唤醒（2026-10-02） +仅更新当前应用时，不写入 NVS、分区表或初始 OTA 元数据。完整安装使用构建产物匹配的 bootloader、分区表及初始化数据，应用仍分段写入。整片擦除或写入包含填充区域的合并镜像会影响已保存的配对和设置。 -用户反馈招手固件在时钟页面很难唤醒。代码上，时钟与普通页面的唯一判定差异是释放要求：OK 或点赞进入时钟后，以及空闲进入时钟引起的上下文变化后，需要连续约 300 ms 检测不到任何手才接受动作；手留在画面内、或有其他目标被检出为手时，挥手和招手都不会被处理。时钟中不显示预览和手部反馈，用户无法判断是否已释放。该原因由代码推断，没有设备日志确认。 +## 维护入口 -- 招手不再要求先释放：做完 OK 或点赞的手不离开画面，直接招手即可唤醒；触摸、按键等外部输入之后的招手同样有效。 -- 单向挥手和 OK、点赞仍要求先释放，避免做完手形后放下手被当作挥手而立刻唤醒。 +应用日志从 UART0 输出：右侧排针 H1 的 15 脚 TX0 / GPIO58，或底板背面 TX 调试焊盘，115200 波特率。Type-C 口使用 USB OTG 下载，摄像头占用的 USB Serial/JTAG 引脚位于左侧排针。日志包含手部坐标、置信度、动作和运行耗时，不包含图像。 -13 条用例在主机通过 ASan/UBSan，OK 用例增加了“手未离开时挥手无效、招手有效”的断言。应用通过 ESP-IDF v6.1 构建。烧录前读取 OTA 数据确认设备从 ota_0 启动；应用按六段写入 ota_0，全部通过设备哈希校验，写入过程无断连，未写入 NVS、分区表、引导程序或 OTA 数据，未请求复位。时钟唤醒率及放下手时的误唤醒待实机验收。 +代码结构、识别参数见[手势设计](../design/mosaico-gesture-input.md)；固定依赖、模型摘要和引脚所有权见[摄像头依赖说明](../../firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md)。 diff --git a/firmware/components/esp-mosaico-bsp/UPSTREAM.md b/firmware/components/esp-mosaico-bsp/UPSTREAM.md index cd830a1..196ad2b 100644 --- a/firmware/components/esp-mosaico-bsp/UPSTREAM.md +++ b/firmware/components/esp-mosaico-bsp/UPSTREAM.md @@ -36,8 +36,9 @@ written; upstream shows undefined frame memory at full brightness during that de `bsp_battery_init()` does not seal the BQ27220 again after `bq27220_create()`. The driver seals and verifies on every path that returns a handle, and each seal waits -`CONFIG_BQ27220_SEAL_SETTLE_MS` (2000 ms) unconditionally, so the upstream call only -added that wait. +`CONFIG_BQ27220_SEAL_SETTLE_MS` unconditionally, so the upstream call only added +that wait. The driver default is 2000 ms; this target selects 200 ms in +`sdkconfig.defaults` and retains the sealed-state check after the wait. When re-syncing, preserve these integration changes as well as the manifest constraint and review the repository diff for other board-specific changes. diff --git a/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md b/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md index de14b6c..766fac5 100644 --- a/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md +++ b/firmware/targets/esp_mosaico/CAMERA_UPSTREAM.md @@ -1,4 +1,4 @@ -# Camera and gesture dependencies +# Experimental camera gestures: dependencies ## Repository reference map @@ -17,9 +17,9 @@ commits below and [dependencies.lock](dependencies.lock), not the latest branch. ## Continuing this implementation -- Read the [interaction design and implementation record](../../../docs/design/mosaico-gesture-input.md) for swipe direction, wake-only behavior, OK hold and protected states. +- Read the [interaction design](../../../docs/design/mosaico-gesture-input.md) for page cycling, wake-only behavior, OK / thumbs-up holds and protected states. - Start in [gesture_camera.cpp](main/gesture_camera.cpp) for capture/pin ownership, [gesture_image.cpp](main/gesture_image.cpp) for image orientation, [gesture_tracker.cpp](main/gesture_tracker.cpp) for temporal recognition, and [gesture_input.cpp](main/gesture_input.cpp) for the worker lifecycle. [app_main.cpp](main/app_main.cpp) owns action routing and UI state. -- Follow the [installation and hardware acceptance guide](../../../docs/validation/mosaico-gestures.md). Software builds/tests passed at the initial implementation; camera orientation, actual recognition, UI layout, resource use and power remain hardware acceptance items. +- Follow the [usage and installation guide](../../../docs/validation/mosaico-gestures.md). Camera gestures are experimental and disabled by default. - Preserve the 7 MiB dual-slot layout, default-off capture setting, local-only image processing and existing BSP display fixes. Recheck the current tree and lockfile before changing dependencies; do not edit generated `managed_components` as the source of truth. ## Locked component versions @@ -32,7 +32,7 @@ The target uses registry components pinned by `dependencies.lock`: | espressif/esp_cam_sensor | 2.6.0 | OV3640 and SC101IOT sensor configuration | | espressif/esp-dl | 3.3.11 | S31 inference and preprocessing | | espressif/hand_detect | 0.2.0 | ESPDet-Pico 224×224 hand detection | -| espressif/hand_gesture_recognition | 0.2.0 | MobileNetV2 128×128 classification; `ok` command | +| espressif/hand_gesture_recognition | 0.2.0 | MobileNetV2 128×128 classification; `ok` and `like` commands | The published model components explicitly select their `models/p4` resources for `esp32s31`. Both models are embedded in the application so A/B update and @@ -76,8 +76,8 @@ SC101IOT produces 320x180 RGB888 before rotation, and OV3640 produces 320x240. The model and debug preview receive the actual rotated dimensions. The two 720p UYVY capture buffers require about 3.52 MiB; the startup precheck requires at least a 9 MiB contiguous free PSRAM block for capture, model arenas and UI load. -The nominal sensor frame rate and close-range recognition require hardware -verification on the CameraBoard's external oscillator. +The configured sensor frame rate differs from the end-to-end recognition rate, +which also includes conversion, inference and scheduling. `CONFIG_MOSAICO_CAMERA_ROTATION` applies the mounting correction after the horizontal mirror; the inverse of the actual applied display rotation follows diff --git a/firmware/targets/esp_mosaico/README.md b/firmware/targets/esp_mosaico/README.md index 30b21b3..a9e7fc6 100644 --- a/firmware/targets/esp_mosaico/README.md +++ b/firmware/targets/esp_mosaico/README.md @@ -36,15 +36,18 @@ eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 - `usage_core` / `usage_protocol` / `usage_ble` / `usage_ota` 与其他目标共用，BLE 广播前缀 `QF-Mosaico-`，状态身份 `ESP-Mosaico`，型号标识 `esp_mosaico`； - 触控标签与左右滑动切页之外，板载 AI 键单击可唤醒并切换到下一页。 -### 本地手势输入 +### 实验性本地手势 -后续开发入口：[参考仓库、固定版本与代码导航](CAMERA_UPSTREAM.md#repository-reference-map)；[交互设计与实施记录](../../../docs/design/mosaico-gesture-input.md)。 +左槽 CameraBoard（OV3640 或 SC101IOT）支持本地手势，默认关闭。在设置中打开 `CAMERA` 并点击 `SAVE` 后启用，关闭并保存即可停止采集。图像只在设备内处理，不上传、不保存；识别效果受光线、距离和手部是否完整入镜影响。 -左槽 CameraBoard（OV3640 或 SC101IOT，自动识别）可选开启本地手势：左挥或上挥上一页、右挥或下挥下一页，时钟中首挥只恢复原页；👌 OK 稳定保持约 500 ms 进入时钟。设置中的 `CAMERA` 开关默认关闭，切换后需 `SAVE`；开关打开时顶栏显示摄像头标识。亮度、自动时钟超时和相机开关作为一条 NVS 记录保存；记录缺失或无效时使用默认显示设置并关闭相机。Type-C 口是 USB OTG，只在 ROM 下载模式下枚举，应用日志只从 UART0（右侧排针 H1 的 15 脚 TX0 / GPIO58，或底板背面调试焊盘 TX）输出。图像仅在本机处理。 +- 用量页：向任意方向挥手或左右招手切到下一页，按总览 → Codex → Claude 循环。 +- 进入时钟：👌 OK 或 👍 点赞稳定保持约半秒。 +- 唤醒：在时钟页面挥手或左右招手，恢复原页面。 +- 预览：点击顶栏摄像头标识显示或隐藏画面与手部定位框；默认隐藏，重启后恢复隐藏。关闭预览不影响识别。 -点击顶部相机图标可显示或隐藏实时预览；预览默认隐藏，重启后恢复隐藏。关闭预览不影响手势识别，摄像头采集仍由设置页 CAMERA 开关控制。时钟、设置及受保护界面不显示预览。预览保留原始画面比例，只显示图像和手部定位框。返回可识别的普通页面后，恢复本次运行中选择的预览状态。 +一段连续挥动只执行一次，手停稳后再做下一次动作。设置、配对和 OTA 期间暂停手势输入，隐藏预览。触摸和 AI 键继续可用；时钟中摄像头仍工作，开启手势会增加耗电。 -模型与代码内嵌同一应用，在当前 7 MiB 双槽分区中共同进行 OTA 更新与回滚。有线更新应用并保留设置时使用分段烧录，不能用填充后的完整镜像替代。依赖来源见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)，安装、恢复与实机验证见[手势验收指南](../../../docs/validation/mosaico-gestures.md)。当前仅通过软件测试和构建；相机方向、识别率、性能、功耗和提示布局仍待设备验收。 +亮度、自动时钟超时和相机开关作为一条 NVS 记录保存；记录缺失或无效时使用默认显示设置并关闭相机。模型随代码内嵌应用，在当前双槽分区中共同更新。使用、安装和故障处理见[手势指南](../../../docs/validation/mosaico-gestures.md)，实现结构见[手势设计](../../../docs/design/mosaico-gesture-input.md)，固定版本、参考仓库和模型摘要见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)。 ### 显示刷新 @@ -56,7 +59,7 @@ TE 信号缺失时最多等待 25 ms 后继续刷新，并在超时、恢复状 启动日志应包含 `partial refresh: PSRAM single buffer`。硬件验收应比较导航条、 用量条动画和整页切换的帧间隔，并检查四个旋转方向、休眠唤醒后的撕裂与触控。 40 MHz 四线 QSPI 传输整屏 RGB565 的理论下限约为 23 ms；整页更新仍可能跨越 -屏幕扫描周期，按帧 TE 同步不等于所有更新均无撕裂。此刷新路径尚待实机验收。 +屏幕扫描周期，按帧 TE 同步不等于所有更新均无撕裂。 ### 与 AMOLED 目标的板级差异 diff --git a/firmware/targets/esp_mosaico/main/app_main.cpp b/firmware/targets/esp_mosaico/main/app_main.cpp index 94865aa..44648c5 100644 --- a/firmware/targets/esp_mosaico/main/app_main.cpp +++ b/firmware/targets/esp_mosaico/main/app_main.cpp @@ -89,8 +89,8 @@ std::atomic ai_button_pressed{false}; PowerMonitor power_monitor; std::atomic power_monitor_ready{false}; -// The gauge driver waits out fixed settle times of several seconds while sealing, so the -// gauge attaches on its own task. The battery stays hidden until it is ready. +// Gauge configuration and settling run asynchronously to avoid delaying BLE and UI startup. +// The main loop must observe the readiness flag before reading the gauge. void power_monitor_task(void* /*context*/) { if (power_monitor.begin()) { @@ -173,11 +173,10 @@ extern "C" void app_main(void) ScreenRotation initial_rotation = kFixedRotation; OrientationTracker orientation_tracker; - // The IMU shares the board I2C bus, which bsp_display_start() brings up - // together with the VCC_3V3 rail, so the panel has to come first. + // Display initialization establishes the shared I2C bus and the IMU's VCC_3V3 supply. ESP_ERROR_CHECK(ui.begin(initial_rotation) ? ESP_OK : ESP_FAIL); - // Started first so the gauge has attached before BMI270 initialization uses the bus. + // Start gauge initialization on the established bus without blocking other peripheral setup. if (xTaskCreate(power_monitor_task, "power_init", 4096, nullptr, 1, nullptr) != pdPASS) { ESP_LOGW(TAG, "BQ27220 start task unavailable; battery will be hidden"); } @@ -185,8 +184,8 @@ extern "C" void app_main(void) // Advertising starts before the remaining peripherals so the bridge can reconnect // while they initialize; link events wait in the queue for the main loop. ESP_ERROR_CHECK(events.begin() ? ESP_OK : ESP_ERR_NO_MEM); - // Wired before BLE so no peer can reach the manifest path ungated; the - // verdict itself is published by the main-loop power poll below. + // Install the power gate before BLE can receive OTA manifests. + // The power verdict remains unavailable until the main-loop poll obtains a valid reading. ota.set_power_source(&ota_power_source); ESP_ERROR_CHECK(ble.start(events, ble_config)); diff --git a/firmware/targets/esp_mosaico/main/gesture_image.hpp b/firmware/targets/esp_mosaico/main/gesture_image.hpp index cb1a961..3c31a9f 100644 --- a/firmware/targets/esp_mosaico/main/gesture_image.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_image.hpp @@ -14,14 +14,30 @@ struct GestureImageSize { unsigned height; }; -/** Fits the complete frame within 320x240, preserving aspect ratio before rotation. - * Returns zero dimensions for invalid input or an unsupported rotation. +/** + * Fits the complete frame within 320x240 before rotation, preserving its aspect ratio + * to integer-pixel precision. Odd quarter-turn counts swap the output dimensions. + * @param width Source width in pixels; must be even and within 2..4096. + * @param height Source height in pixels; must be within 1..4096. + * @param quarter_turns Clockwise rotation in 90-degree steps; valid range is 0..3. + * @return Output dimensions, or zero dimensions if any argument is invalid. */ GestureImageSize gesture_image_size(unsigned width, unsigned height, unsigned quarter_turns); -/** Downsamples packed UYVY to RGB888, mirrors horizontally, then rotates clockwise. - * Output capacity must be at least kGestureImageBytes. Actual packed dimensions - * are returned by gesture_image_size(); no cropping or padding is applied. +/** + * Converts the complete UYVY frame to RGB888 using gesture_image_size() dimensions, + * then applies horizontal mirroring and clockwise rotation. Output has no row padding. + * Both buffers remain caller-owned and must not overlap; neither pointer is retained. + * @param src Non-null source buffer containing packed UYVY rows with optional row padding. + * @param size Source buffer size in bytes; must be at least stride * height. + * @param width Source width in pixels; must be even and within 2..4096. + * @param height Source height in pixels; must be within 1..4096. + * @param stride Source row stride in bytes; must be at least width * 2. + * @param dst Non-null output buffer with at least kGestureImageBytes capacity; capacity is not checked. + * @param quarter_turns Clockwise rotation in 90-degree steps; valid range is 0..3. + * @param mirror Whether to mirror horizontally before rotation. + * @return True after conversion; false for null pointers, invalid dimensions or rotation, + * insufficient source size, or an invalid stride. Validation failure leaves dst unchanged. */ bool prepare_gesture_image(const uint8_t* src, size_t size, unsigned width, unsigned height, unsigned stride, uint8_t* dst, diff --git a/firmware/targets/esp_mosaico/main/gesture_input.cpp b/firmware/targets/esp_mosaico/main/gesture_input.cpp index 2dff98d..dd0b458 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.cpp @@ -101,8 +101,8 @@ void GestureInput::run() uint64_t measured_us = 0; uint64_t capture_us = 0, detect_us = 0, classify_us = 0, feedback_us = 0; unsigned warmup = 0; - // Start-up failures scroll past during boot, so the reason is repeated - // after cleanup until the switch is turned off. + // Retain failure details for periodic diagnostics while recognition is disabled. + // Disabling capture clears recoverable faults; cleanup faults persist until reboot. const char* failure = nullptr; int failure_code = 0; uint64_t next_failure_log = 0; diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp index d2d6e2d..b336e9b 100644 --- a/firmware/targets/esp_mosaico/main/gesture_tracker.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.cpp @@ -5,25 +5,22 @@ namespace usage_panel::mosaico { namespace { -// Dropouts up to this long keep a swipe or OK hold alive. A longer gap restarts -// the track from the next usable sample; it does not require the hand to leave. -// Frames arrive about every 125 ms and slower while the OK classifier or the -// preview runs, so the limit must exceed one frame interval at that cadence. +// Short detection gaps preserve the current swipe or pose hold. Longer gaps restart +// tracking at the next usable sample. The limit allows for classification and preview latency. constexpr uint64_t kMaxSampleGapMs = 400; -// After a swipe or wave, motion must cease for this long before the next command, so the +// After a swipe or wave, motion must cease for this long before the next swipe or wave, so the // follow-through, an immediate return and the rest of a longer wave cannot repeat it. constexpr uint64_t kSettleMs = 400; -// A swipe opposite to the previous one within this time is the hand returning. +// An opposite swipe within this interval is treated as the return stroke. constexpr uint64_t kReturnStrokeMs = 1500; constexpr uint64_t kOkHoldMs = 500; -// Samples of one hold are at most kMaxSampleGapMs apart, so a completed hold spans three. +// A completed hold requires at least three samples because each gap is shorter than the hold. static_assert(kMaxSampleGapMs < kOkHoldMs); -// Frame-to-frame movement up to this distance is detector jitter around a resting hand. +// Frame-to-frame movement up to this distance is treated as detector jitter. constexpr float kRestStep = 0.03F; -// Beyond this frame-to-frame movement the hand is mid-stroke and its pose is not classified. +// Larger frame-to-frame movement skips pose classification to preserve motion sampling cadence. constexpr float kSteadyStep = 0.05F; -// A wave is three alternating horizontal legs of at least this length. Sampling at about -// 8 fps sees roughly two thirds of the true travel between reversals. +// Each of the three alternating horizontal wave legs must cover this fraction of the image width. constexpr float kWaveLeg = 0.12F; GestureAction opposite(GestureAction action) @@ -43,7 +40,7 @@ GestureEffect route_gesture(GestureAction action, bool allowed, bool asleep) if (!allowed || action == GestureAction::None) return GestureEffect::None; if (action == GestureAction::EnterClock) return asleep ? GestureEffect::None : GestureEffect::Clock; if (asleep) return GestureEffect::Wake; - // Every direction advances, so the tracker's return-stroke window never drops a command. + // Swipes and waves share forward-only navigation. return GestureEffect::Next; } @@ -83,7 +80,7 @@ void GestureTracker::begin_stroke(const HandObservation& h, uint64_t now) GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) { if (h.count > 1) { - // Several hands make the command owner ambiguous; wait for the view to clear. + // Multiple hands invalidate ownership and require release before another swipe or pose. absent_ = false; tracking_ = holding_ok_ = false; waiting_release_ = true; @@ -154,8 +151,8 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) GestureAction action = GestureAction::None; if (!settling_) { - // Until the hand has left the view, a stroke may be the hand withdrawing after a pose - // or a touch. A wave cannot be, so it alone is accepted without that release. + // Hand withdrawal after a pose or touch can resemble a swipe, so swipes require release. + // Waves use alternating horizontal motion as their confirmation criterion. if (!waiting_release_ && samples_ >= 3 && now - started_ >= 150 && now - started_ <= 900) { if (std::fabs(dx) >= 0.25F && std::fabs(dx) >= 2 * max_dy_ && max_dy_ <= 0.15F && std::fabs(dx) >= path_x_ * 0.75F) { @@ -190,7 +187,7 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) holding_ok_ = false; } if (action != GestureAction::None) { - // Lowering the hand after OK must not wake the clock, so only OK waits for release. + // Clock poses require release before another swipe or pose to suppress withdrawal-triggered wakeups. waiting_release_ = action == GestureAction::EnterClock; last_swipe_ = waiting_release_ ? GestureAction::None : action; last_action_ = last_motion_ = now; @@ -200,7 +197,7 @@ GestureAction GestureTracker::update(const HandObservation& h, uint64_t now) leg_x_ = 0; legs_ = 0; } else if (now - started_ > 900) { - // Motion that qualified as nothing so far must not hold back a later stroke. + // Restart expired candidates so subsequent motion can form a new stroke. begin_stroke(h, now); } return action; diff --git a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp index 70af87a..d4984a2 100644 --- a/firmware/targets/esp_mosaico/main/gesture_tracker.hpp +++ b/firmware/targets/esp_mosaico/main/gesture_tracker.hpp @@ -22,8 +22,8 @@ struct HandObservation { }; /** - * True when one hand moved little enough between consecutive frames for its pose to be - * classified. Classifying a moving hand lengthens the frame interval and costs a stroke samples. + * Returns whether consecutive single-hand observations satisfy the pose-classification + * motion threshold. Skipping classification during motion preserves trajectory sampling cadence. */ bool hand_is_steady(const HandObservation& previous, const HandObservation& current); From b58ef7fcef7adeb1317f180c438ce7762f345e98 Mon Sep 17 00:00:00 2001 From: wulu Date: Fri, 2 Oct 2026 22:54:23 +0800 Subject: [PATCH 13/15] docs(mosaico): align license record, comments and doc index Record espressif/dhara 1.1.0 to match dependencies.lock; the distributed license text is unchanged from 1.0.0. Describe the gesture context word layout, and state that the first completed refresh starts the brightness fade-in rather than begin(). Index the Mosaico boot flow document, list camera gestures in the verified device coverage, and describe camera start as up to three attempts, matching the code. --- THIRD_PARTY_LICENSES.md | 6 +++--- docs/design/mosaico-gesture-input.md | 2 +- docs/roadmap.md | 1 + docs/verification.md | 2 +- firmware/targets/esp_mosaico/main/display_ui.cpp | 2 +- firmware/targets/esp_mosaico/main/gesture_input.cpp | 2 ++ 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index 5f11724..b6a5f7a 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -528,10 +528,10 @@ SOFTWARE. ## `espressif/dhara` -- Version: `1.0.0` -- Source: https://components.espressif.com/components/espressif/dhara/versions/1.0.0 +- Version: `1.1.0` +- Source: https://components.espressif.com/components/espressif/dhara/versions/1.1.0 - SPDX conclusion from distributed license: `ISC` -- Copyright/notice: https://components.espressif.com/components/espressif/dhara/versions/1.0.0/license +- Copyright/notice: https://components.espressif.com/components/espressif/dhara/versions/1.1.0/license ## `espressif/esp_lcd_touch_cst9220` diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md index 3c92aa9..f22b68c 100644 --- a/docs/design/mosaico-gesture-input.md +++ b/docs/design/mosaico-gesture-input.md @@ -44,7 +44,7 @@ CameraBoard / esp_video 主循环拥有页面和屏保状态。事件队列容量为 1，满时丢弃，不阻塞推理。每个事件带时间戳和上下文代次；超过 300 ms 或代次不匹配时丢弃。触摸、按键、连接变化、页面、旋转和输入许可变化使旧轨迹失效。被接受的手势会保留自身的重复触发抑制状态。 -关闭相机或进入受保护状态时，视觉任务完成当前帧后停止流并释放模型和缓冲。启动失败最多重试三次；连续三次采集失败停用识别。普通故障可通过关闭并重新开启重试；无法安全释放驱动资源时，需重启设备。主循环的触摸、按键和 BLE 路径独立于视觉任务。 +关闭相机或进入受保护状态时，视觉任务完成当前帧后停止流并释放模型和缓冲。启动最多尝试三次；连续三次采集失败停用识别。普通故障可通过关闭并重新开启重试；无法安全释放驱动资源时，需重启设备。主循环的触摸、按键和 BLE 路径独立于视觉任务。 ## 识别参数 diff --git a/docs/roadmap.md b/docs/roadmap.md index f6c9be7..1a68ac6 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -18,6 +18,7 @@ | [软件与发布验收](validation/software-acceptance.md) | 测试入口与发布候选验收清单 | | [AMOLED 性能参考](validation/amoled-performance.md) | 内存、刷新耗时与测量条件 | | [ePaper 刷新参考](validation/epaper-refresh.md) | 局刷数据及光学验证边界 | +| [Mosaico 启动流程](validation/mosaico-boot.md) | 显示、连接与电量计初始化顺序 | | [接入架构](design/open-device-access.md) | 状态归属、连接与更新生命周期 | | [Mosaico 实验性手势](design/mosaico-gesture-input.md) | 操作规则、处理链路与识别参数 | | [接入协议](../protocol/open-device-access.md) | 设备线上字段与校验约束 | diff --git a/docs/verification.md b/docs/verification.md index 4af5683..dc2737d 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -9,7 +9,7 @@ | M5StickS3 | 连接、用量显示、页面切换；OTA 接收超时修复后连续 5 次正常升级；电池供电下电源门槛拒绝 | 40% 电量边界、配置变体、故障注入与长期稳定性 | | Waveshare AMOLED 2.16 | 连接、用量显示、页面与时钟切换、正常 OTA；电池供电下电源门槛拒绝 | 40% 边界、RTC 断电保持、唤醒配置、电池与长期屏保 | | Waveshare ePaper 3.97 | 连接、用量显示、页面与时钟切换、本地工具正常 OTA | 多日残影、不同温度/方向、SD 读写与断电恢复 | -| ESP-Mosaico | 连接、用量显示、页面与时钟切换、本地工具正常 OTA；拔除外部电源后拒绝更新 | 电量边界、插电识别延迟、续航与故障注入 | +| ESP-Mosaico | 连接、用量显示、页面与时钟切换、本地工具正常 OTA；拔除外部电源后拒绝更新；实验性摄像头手势 | 电量边界、插电识别延迟、续航与故障注入 | 以上基础功能和正常 OTA 覆盖来自 2026-09 的开发设备测试，未固定同一套发布候选资产。脚本 OTA 证明设备升级路径，不涵盖安装版 Bridge 的下载、桌面确认和独占连接调度。 diff --git a/firmware/targets/esp_mosaico/main/display_ui.cpp b/firmware/targets/esp_mosaico/main/display_ui.cpp index 63a2577..8e778e1 100644 --- a/firmware/targets/esp_mosaico/main/display_ui.cpp +++ b/firmware/targets/esp_mosaico/main/display_ui.cpp @@ -152,7 +152,7 @@ lv_display_t* start_display( ESP_LOGE(TAG, "touch initialization failed"); return nullptr; } - // The panel comes up at zero brightness; begin() raises it after the first frame. + // The panel comes up at zero brightness; the first completed refresh starts the fade-in. return display; } diff --git a/firmware/targets/esp_mosaico/main/gesture_input.cpp b/firmware/targets/esp_mosaico/main/gesture_input.cpp index dd0b458..9b84308 100644 --- a/firmware/targets/esp_mosaico/main/gesture_input.cpp +++ b/firmware/targets/esp_mosaico/main/gesture_input.cpp @@ -22,6 +22,8 @@ namespace usage_panel::mosaico { static_assert(CONFIG_MOSAICO_CAMERA_ROTATION % 90 == 0, "Camera rotation must be a multiple of 90 degrees"); #endif namespace { +// context_ layout: bits 0..2 hold the flags below, bits 3..4 the display rotation, +// bits 5..6 the page, bit 7 marks an accepted gesture and bits 8..31 count generations. constexpr uint32_t kEnabled = 1; constexpr uint32_t kAllowed = 2; constexpr uint32_t kAsleep = 4; From 7d83ae84613eabbfcdf138d62e3da8b51586e13f Mon Sep 17 00:00:00 2001 From: wulu Date: Fri, 2 Oct 2026 23:05:05 +0800 Subject: [PATCH 14/15] fix(mosaico): pin the logic test target to esp32s31 Without sdkconfig.defaults a clean configure selects the default esp32 target, whose DRAM cannot hold the 230400-byte gesture image buffer used by the conversion tests, so the test image fails to link. The other test applications already pin their target the same way. --- firmware/targets/esp_mosaico/test_apps/logic/sdkconfig.defaults | 1 + 1 file changed, 1 insertion(+) create mode 100644 firmware/targets/esp_mosaico/test_apps/logic/sdkconfig.defaults diff --git a/firmware/targets/esp_mosaico/test_apps/logic/sdkconfig.defaults b/firmware/targets/esp_mosaico/test_apps/logic/sdkconfig.defaults new file mode 100644 index 0000000..3a329c6 --- /dev/null +++ b/firmware/targets/esp_mosaico/test_apps/logic/sdkconfig.defaults @@ -0,0 +1 @@ +CONFIG_IDF_TARGET="esp32s31" From 0c61451841b8f15fe811b6b34134ab627e99f342 Mon Sep 17 00:00:00 2001 From: wulu Date: Sat, 3 Oct 2026 07:48:34 +0800 Subject: [PATCH 15/15] fix(mosaico): keep display preferences saved by earlier firmware Earlier firmware stored brightness and the auto-clock timeout as separate keys in the shared "display" NVS namespace. When the combined mosaico/settings record is missing or invalid, load those keys before falling back to defaults, so an upgrade that preserves NVS keeps the user's preferences. Capture stays off, and the next save writes the combined record. --- docs/design/mosaico-gesture-input.md | 2 +- firmware/targets/esp_mosaico/README.md | 2 +- .../esp_mosaico/main/panel_settings.cpp | 6 ++-- .../esp_mosaico/main/panel_settings.hpp | 5 +++- .../logic/main/test_panel_settings.cpp | 28 +++++++++++++++++++ 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/docs/design/mosaico-gesture-input.md b/docs/design/mosaico-gesture-input.md index f22b68c..c2ee205 100644 --- a/docs/design/mosaico-gesture-input.md +++ b/docs/design/mosaico-gesture-input.md @@ -14,7 +14,7 @@ 触摸滑动保留双向导航。摄像头唤醒依赖持续采集和识别，时钟屏保不是芯片深度睡眠。 -设置页的 `CAMERA` 开关与亮度、自动时钟超时一并保存。调整开关只改变设置预览，`SAVE` 成功后才改变采集状态，`CANCEL` 恢复保存值。NVS 使用 `mosaico/settings` 单个整数保存三个字段；记录缺失或无效时使用默认显示设置并关闭相机。 +设置页的 `CAMERA` 开关与亮度、自动时钟超时一并保存。调整开关只改变设置预览，`SAVE` 成功后才改变采集状态，`CANCEL` 恢复保存值。NVS 使用 `mosaico/settings` 单个整数保存三个字段；记录缺失或无效时关闭相机，亮度和自动时钟超时沿用旧版固件在 `display` 命名空间中保存的值，没有则使用默认值。 图像仅在本机处理，不上传或保存。点击摄像头标识切换预览，预览选择不持久化。界面尺寸与状态见[AMOLED 设计规范](amoled-ui.md)。 diff --git a/firmware/targets/esp_mosaico/README.md b/firmware/targets/esp_mosaico/README.md index a9e7fc6..1858f20 100644 --- a/firmware/targets/esp_mosaico/README.md +++ b/firmware/targets/esp_mosaico/README.md @@ -47,7 +47,7 @@ eim run "idf.py -C firmware/targets/esp_mosaico build" v6.1 一段连续挥动只执行一次，手停稳后再做下一次动作。设置、配对和 OTA 期间暂停手势输入，隐藏预览。触摸和 AI 键继续可用；时钟中摄像头仍工作，开启手势会增加耗电。 -亮度、自动时钟超时和相机开关作为一条 NVS 记录保存；记录缺失或无效时使用默认显示设置并关闭相机。模型随代码内嵌应用，在当前双槽分区中共同更新。使用、安装和故障处理见[手势指南](../../../docs/validation/mosaico-gestures.md)，实现结构见[手势设计](../../../docs/design/mosaico-gesture-input.md)，固定版本、参考仓库和模型摘要见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)。 +亮度、自动时钟超时和相机开关作为一条 NVS 记录保存；记录缺失或无效时关闭相机，亮度和自动时钟超时沿用旧版固件保存的值，没有则使用默认值。模型随代码内嵌应用，在当前双槽分区中共同更新。使用、安装和故障处理见[手势指南](../../../docs/validation/mosaico-gestures.md)，实现结构见[手势设计](../../../docs/design/mosaico-gesture-input.md)，固定版本、参考仓库和模型摘要见 [CAMERA_UPSTREAM.md](CAMERA_UPSTREAM.md)。 ### 显示刷新 diff --git a/firmware/targets/esp_mosaico/main/panel_settings.cpp b/firmware/targets/esp_mosaico/main/panel_settings.cpp index f48e4db..91e7c23 100644 --- a/firmware/targets/esp_mosaico/main/panel_settings.cpp +++ b/firmware/targets/esp_mosaico/main/panel_settings.cpp @@ -20,14 +20,16 @@ bool valid(const DisplaySettings& settings) PanelSettings load_panel_settings(DisplaySettings defaults) { + // Firmware without camera gestures stored brightness and timeout as separate + // keys in the shared "display" namespace; they apply until the next save. nvs_handle_t handle; - if (nvs_open(kNamespace, NVS_READONLY, &handle) != ESP_OK) return {defaults}; + if (nvs_open(kNamespace, NVS_READONLY, &handle) != ESP_OK) return {load_display_settings(defaults)}; uint32_t packed = 0; const esp_err_t error = nvs_get_u32(handle, kSettingsKey, &packed); nvs_close(handle); const DisplaySettings display{static_cast(packed & kBrightnessMask), packed >> kTimeoutShift}; - if (error != ESP_OK || !valid(display)) return {defaults}; + if (error != ESP_OK || !valid(display)) return {load_display_settings(defaults)}; return {display, (packed & kCameraMask) != 0}; } diff --git a/firmware/targets/esp_mosaico/main/panel_settings.hpp b/firmware/targets/esp_mosaico/main/panel_settings.hpp index b0c06c7..52dc734 100644 --- a/firmware/targets/esp_mosaico/main/panel_settings.hpp +++ b/firmware/targets/esp_mosaico/main/panel_settings.hpp @@ -9,7 +9,10 @@ struct PanelSettings { bool camera_enabled = false; }; -/** Loads one settings record; missing or invalid records retain defaults and disable capture. */ +/** + * Loads one settings record. A missing or invalid record disables capture and falls back to + * the separate display keys written by earlier firmware, then to defaults for each field. + */ PanelSettings load_panel_settings(DisplaySettings defaults); /** * Saves all fields in one NVS value. Brightness must be 1..100 and timeout 0..86400 seconds. diff --git a/firmware/targets/esp_mosaico/test_apps/logic/main/test_panel_settings.cpp b/firmware/targets/esp_mosaico/test_apps/logic/main/test_panel_settings.cpp index 9f70003..f168d12 100644 --- a/firmware/targets/esp_mosaico/test_apps/logic/main/test_panel_settings.cpp +++ b/firmware/targets/esp_mosaico/test_apps/logic/main/test_panel_settings.cpp @@ -50,3 +50,31 @@ TEST_CASE("missing or invalid panel records use defaults with capture off", "[se TEST_ASSERT_EQUAL(60, actual.display.clock_timeout_seconds); TEST_ASSERT_FALSE(actual.camera_enabled); } + +TEST_CASE("display keys from earlier firmware apply until a panel record is saved", "[settings]") +{ + TEST_ASSERT_EQUAL(ESP_OK, nvs_flash_init()); + nvs_handle_t handle; + TEST_ASSERT_EQUAL(ESP_OK, nvs_open("mosaico", NVS_READWRITE, &handle)); + const esp_err_t erased = nvs_erase_key(handle, "settings"); + TEST_ASSERT_TRUE(erased == ESP_OK || erased == ESP_ERR_NVS_NOT_FOUND); + TEST_ASSERT_EQUAL(ESP_OK, nvs_commit(handle)); + nvs_close(handle); + TEST_ASSERT_EQUAL(ESP_OK, save_display_settings({70, 600})); + + auto actual = load_panel_settings({40, 300}); + TEST_ASSERT_EQUAL(70, actual.display.brightness); + TEST_ASSERT_EQUAL(600, actual.display.clock_timeout_seconds); + TEST_ASSERT_FALSE(actual.camera_enabled); + + TEST_ASSERT_EQUAL(ESP_OK, save_panel_settings({{20, 60}, true})); + actual = load_panel_settings({40, 300}); + TEST_ASSERT_EQUAL(20, actual.display.brightness); + TEST_ASSERT_EQUAL(60, actual.display.clock_timeout_seconds); + TEST_ASSERT_TRUE(actual.camera_enabled); + + TEST_ASSERT_EQUAL(ESP_OK, nvs_open("display", NVS_READWRITE, &handle)); + TEST_ASSERT_EQUAL(ESP_OK, nvs_erase_all(handle)); + TEST_ASSERT_EQUAL(ESP_OK, nvs_commit(handle)); + nvs_close(handle); +}