From 661f1f9df76fae6cd83cd58be08e3503255b47c9 Mon Sep 17 00:00:00 2001 From: zhouziying Date: Wed, 1 Jul 2026 16:21:23 +0800 Subject: [PATCH] feat: support pro2 reset & import --- electron/main.ts | 48 +- electron/mcp/index.ts | 11 +- electron/mcp/ocrScenes.ts | 6 + electron/mcp/pro2SequenceResolver.ts | 97 ++ electron/mcp/pro2Sequences.ts | 1513 +++++++++++++++++++++++++ electron/mcp/sequenceSetResolver.ts | 42 + electron/mcp/sequenceSets.ts | 60 + electron/mcp/sequences.ts | 20 +- electron/mcp/state.ts | 10 +- electron/mcp/tools/executeSequence.ts | 43 +- electron/paddleOcrEn.ts | 1 + electron/preload.ts | 21 +- scripts/paddleocr_en_infer.py | 111 +- src/components/CameraPanel.tsx | 195 +++- src/components/ControlPanel.css | 47 + src/components/ControlPanel.tsx | 247 +++- src/deviceTestSetPreference.ts | 24 + src/ocr/deviceScenes.ts | 55 + src/vite-env.d.ts | 13 +- 19 files changed, 2405 insertions(+), 159 deletions(-) create mode 100644 electron/mcp/ocrScenes.ts create mode 100644 electron/mcp/pro2SequenceResolver.ts create mode 100644 electron/mcp/pro2Sequences.ts create mode 100644 electron/mcp/sequenceSetResolver.ts create mode 100644 electron/mcp/sequenceSets.ts create mode 100644 src/deviceTestSetPreference.ts create mode 100644 src/ocr/deviceScenes.ts diff --git a/electron/main.ts b/electron/main.ts index 56fa899..e2176cb 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -22,7 +22,7 @@ import { type VerifyOcrResult, } from './mcp/state'; import { saveCaptureToDownloads } from './saveCapture'; -import { resolveSequenceStepsById } from './mcp/sequenceResolver'; +import { resolveSequenceStepsByIdForDevice } from './mcp/sequenceSetResolver'; /** * Build output directory structure: @@ -54,8 +54,18 @@ let hasCompletedArmCleanupBeforeQuit = false; /** Use bracket notation to avoid vite:define plugin transformation */ const VITE_DEV_SERVER_URL = process.env['VITE_DEV_SERVER_URL']; +const shouldUseSingleInstanceLock = !VITE_DEV_SERVER_URL; +const gotSingleInstanceLock = shouldUseSingleInstanceLock + ? app.requestSingleInstanceLock() + : true; + +if (shouldUseSingleInstanceLock && !gotSingleInstanceLock) { + app.quit(); +} + interface RendererSequenceExecutionRequestPayload { sequenceId: string; + deviceTestSetId?: string; armState: { isConnected: boolean; resourceHandle: number; @@ -365,7 +375,8 @@ async function runMnemonicOcrFromRenderer( } async function runSequenceInRenderer( - sequenceId: string + sequenceId: string, + deviceTestSetId?: string ): Promise { if (!mainWindow) { console.warn('Cannot run sequence in renderer: mainWindow is null'); @@ -375,6 +386,7 @@ async function runSequenceInRenderer( const armState = getArmState(); const payload: RendererSequenceExecutionRequestPayload = { sequenceId, + deviceTestSetId, armState: { isConnected: armState.isConnected, resourceHandle: armState.resourceHandle, @@ -468,7 +480,24 @@ async function startMcpServer(): Promise { * Starts MCP Server for Claude/Cursor integration. * On macOS, recreates window when dock icon is clicked with no windows open. */ +if (shouldUseSingleInstanceLock && gotSingleInstanceLock) { + app.on('second-instance', () => { + if (!mainWindow) { + return; + } + + if (mainWindow.isMinimized()) { + mainWindow.restore(); + } + mainWindow.focus(); + }); +} + app.whenReady().then(async () => { + if (!gotSingleInstanceLock) { + return; + } + createWindow(); // Start MCP Server after window is created @@ -495,10 +524,20 @@ app.on('window-all-closed', () => { /** Clean up MCP Server before quitting */ app.on('before-quit', (event) => { + if (shouldUseSingleInstanceLock && !gotSingleInstanceLock) { + return; + } + if (hasCompletedArmCleanupBeforeQuit) { return; } + const armState = getArmState(); + if (!armState.isConnected || armState.resourceHandle <= 0) { + resetArmState(); + return; + } + event.preventDefault(); void disconnectArmController('application quit').finally(() => { hasCompletedArmCleanupBeforeQuit = true; @@ -536,8 +575,8 @@ ipcMain.handle('http-request', async (_event, url: string) => { return performNetRequest(url, 'renderer-http-request'); }); -ipcMain.handle('resolve-sequence-steps', async (_event, sequenceId: string) => { - return resolveSequenceStepsById(sequenceId); +ipcMain.handle('resolve-sequence-steps', async (_event, sequenceId: string, deviceTestSetId?: string) => { + return resolveSequenceStepsByIdForDevice(sequenceId, deviceTestSetId); }); ipcMain.handle( @@ -637,6 +676,7 @@ ipcMain.handle( imageDataUrl: string; layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic'; expectedWordCount?: number; + recVariantCount?: number; } ) => runPaddleOcrEn(payload) ); diff --git a/electron/mcp/index.ts b/electron/mcp/index.ts index 1dba65a..2050174 100644 --- a/electron/mcp/index.ts +++ b/electron/mcp/index.ts @@ -43,7 +43,7 @@ import { executeMnemonicVerify, } from './tools'; import { ARM_STATUS_URI, getArmStatusResource } from './resources'; -import { getAllSequenceIds } from './sequences'; +import { DEVICE_TEST_SETS, getAllSequenceIds } from './sequenceSets'; import { sendMcpLog } from './state'; /** MCP Server configuration */ @@ -257,7 +257,11 @@ export class QAAutoHardwareMcpServer { inputSchema: executeSequenceSchema, }, async (args) => { - sendMcpLog({ type: 'request', action: 'execute-sequence', detail: `Sequence: ${args.sequenceId}` }); + sendMcpLog({ + type: 'request', + action: 'execute-sequence', + detail: `${args.deviceTestSetId ?? 'pro'} / ${args.sequenceId}`, + }); const { output, frame } = await executeExecuteSequence(args, this.httpRequest); sendMcpLog({ type: output.success ? 'response' : 'error', @@ -673,6 +677,9 @@ export class QAAutoHardwareMcpServer { message: ocr.message, ocr, sequenceIds: getAllSequenceIds(), + sequenceSets: Object.fromEntries( + DEVICE_TEST_SETS.map((set) => [set.id, getAllSequenceIds(set.id)]) + ), activeSessions: { streamable: this.streamableTransports.size, sse: this.sseTransports.size, diff --git a/electron/mcp/ocrScenes.ts b/electron/mcp/ocrScenes.ts new file mode 100644 index 0000000..534e08f --- /dev/null +++ b/electron/mcp/ocrScenes.ts @@ -0,0 +1,6 @@ +export { + DEVICE_OCR_SCENES, + PRO2_OCR_SCENES, + type DeviceOcrScenes, + type SequenceOcrSceneConfig, +} from '../../src/ocr/deviceScenes'; diff --git a/electron/mcp/pro2SequenceResolver.ts b/electron/mcp/pro2SequenceResolver.ts new file mode 100644 index 0000000..640db68 --- /dev/null +++ b/electron/mcp/pro2SequenceResolver.ts @@ -0,0 +1,97 @@ +import fs from 'fs'; +import path from 'path'; +import { + generateSlip39ShareSteps, + generateWordSteps, + getPageAction, + getSequence, + pickRandomShares, + type AutoSequence, + type AutoStep, + type MnemonicSource, + type PageAction, +} from './pro2Sequences'; + +interface MnemonicsData { + bip39: Record; + slip39: Record; +} + +function loadMnemonics(): MnemonicsData { + const candidates = [ + path.join(process.cwd(), 'mnemonics.local.json'), + path.join(__dirname, '..', '..', 'mnemonics.local.json'), + ]; + + for (const filePath of candidates) { + if (fs.existsSync(filePath)) { + try { + const raw = fs.readFileSync(filePath, 'utf-8'); + return JSON.parse(raw) as MnemonicsData; + } catch (err) { + console.error(`[mnemonics] Failed to parse ${filePath}:`, err); + } + } + } + + console.warn( + '[mnemonics] mnemonics.local.json not found. Import-wallet sequences will use placeholder words.\n' + + ' Copy mnemonics.example.json -> mnemonics.local.json and fill in your test mnemonics.' + ); + return { bip39: {}, slip39: {} }; +} + +const MNEMONICS = loadMnemonics(); + +function getMnemonicWords(section: 'bip39' | 'slip39', key: string): string[] { + const phrase = MNEMONICS[section]?.[key] ?? ''; + return phrase ? phrase.split(' ') : []; +} + +function resolveMnemonicSource(source: MnemonicSource): AutoStep[] { + const shares = source.keys.map((key) => getMnemonicWords(source.section, key)); + + switch (source.mode) { + case 'single': + return generateWordSteps(shares[0] ?? []); + case 'shares-all': + return generateSlip39ShareSteps(shares); + case 'shares-random': + return generateSlip39ShareSteps(pickRandomShares(shares, source.pickCount ?? 1)); + default: + return []; + } +} + +export function resolvePro2PageActionSteps(action: PageAction): AutoStep[] { + if (action.mnemonicSource) { + return resolveMnemonicSource(action.mnemonicSource); + } + if (action.buildSteps) { + return action.buildSteps(); + } + return action.steps; +} + +export function resolvePro2SequenceSteps(sequence: AutoSequence): AutoStep[] { + const steps: AutoStep[] = []; + + for (const actionId of sequence.actions) { + const action = getPageAction(actionId); + if (!action) { + console.warn(`[pro2-sequence-resolver] Unknown page action ID: ${actionId}`); + continue; + } + steps.push(...resolvePro2PageActionSteps(action)); + } + + return steps; +} + +export function resolvePro2SequenceStepsById(sequenceId: string): AutoStep[] { + const sequence = getSequence(sequenceId); + if (!sequence) { + throw new Error(`Unknown Pro2 sequence ID: ${sequenceId}`); + } + return resolvePro2SequenceSteps(sequence); +} diff --git a/electron/mcp/pro2Sequences.ts b/electron/mcp/pro2Sequences.ts new file mode 100644 index 0000000..7251936 --- /dev/null +++ b/electron/mcp/pro2Sequences.ts @@ -0,0 +1,1513 @@ +/** + * Pro2 Auto Operation Sequences + * + * This file is intentionally separate from sequences.ts (Pro). + * Pro2 starts with the same test items as Pro, but its coordinates and flows + * should be maintained here only while re-recording the new device. + */ +import { PRO2_OCR_SCENES, type SequenceOcrSceneConfig } from './ocrScenes'; + +/** Represents a single step in the auto operation sequence */ +export interface OcrCaptureOptions { + /** Expected mnemonic word count for this capture (12/18/20/24). */ + expectedWordCount?: number; + /** Merge this capture with already stored mnemonic words by index. */ + mergeWithStored?: boolean; + /** Allow partial OCR result (used by first page of 24-word capture). */ + allowPartial?: boolean; + /** Whether checksum validation is required for this capture to be treated as success. */ + requireBip39?: boolean; + /** Optional OCR crop override for device-specific camera framing. */ + sceneConfig?: SequenceOcrSceneConfig; +} + +export interface AutoStep { + label: string; + x: number; + y: number; + depth: number; + /** Optional delay in ms before this step executes. */ + delayBefore?: number; + /** Optional delay in ms after this step (default: 200ms) */ + delayAfter?: number; + /** If set, performs a swipe from (x,y) to swipeTo coordinates instead of a click */ + swipeTo?: { x: number; y: number }; + /** Optional segmented swipe count (>=1). Higher value means slower movement on screen. */ + swipeSegments?: number; + /** Delay in ms between segmented swipe points (default: 0ms). */ + swipeSegmentDelay?: number; + /** Delay in ms before raising stylus after swipe (default: 50ms) */ + swipeHoldDelay?: number; + /** If set, moves arm to position without clicking. */ + moveOnly?: boolean; + /** If set, moves arm to position without clicking, then triggers OCR capture. */ + ocrCapture?: boolean | OcrCaptureOptions; + /** If set, performs verification OCR and clicks the correct option */ + ocrVerify?: { + options: { x: number; y: number; depth: number }[]; + }; +} + +/** + * PageAction: an atomic, reusable page-level operation. + * Each action represents one logical interaction on a device page + * (e.g., "select language", "enter PIN", "click create wallet"). + * Actions can be freely composed into sequences. + */ +export interface PageAction { + id: string; + name: string; + /** Logical group for organization (e.g., '初始设置', '钱包路径') */ + group: string; + /** Steps that make up this action */ + steps: AutoStep[]; + /** Optional dynamic step builder evaluated when the sequence runs. */ + buildSteps?: () => AutoStep[]; + /** Optional mnemonic source to be resolved in Node/Electron main before execution. */ + mnemonicSource?: MnemonicSource; +} + +export interface MnemonicSource { + section: 'bip39' | 'slip39'; + keys: string[]; + mode: 'single' | 'shares-all' | 'shares-random'; + pickCount?: number; +} + +// ============================================================================ +// Keyboard coordinate mapping +// ============================================================================ + +/** Keyboard letter coordinates */ +export const LETTER_COORDS: Record = { + q: { x: 192, y: 75 }, w: { x: 197, y: 75 }, e: { x: 202, y: 75 }, r: { x: 206, y: 75 }, + t: { x: 210, y: 75 }, y: { x: 215, y: 75 }, u: { x: 220, y: 75 }, i: { x: 224, y: 75 }, + o: { x: 228, y: 75 }, p: { x: 232, y: 75 }, + a: { x: 194, y: 82 }, s: { x: 199, y: 82 }, d: { x: 203, y: 82 }, f: { x: 208, y: 82 }, + g: { x: 212, y: 82 }, h: { x: 217, y: 82 }, j: { x: 222, y: 82 }, k: { x: 227, y: 82 }, + l: { x: 231, y: 82 }, + z: { x: 198, y: 91 }, x: { x: 203, y: 91 }, c: { x: 208, y: 91 }, v: { x: 213, y: 91 }, + b: { x: 218, y: 91 }, n: { x: 222, y: 91 }, m: { x: 226, y: 91 }, +}; + +/** Number coordinates on PIN pad */ +export const NUMBER_COORDS: Record = { + '1': { x: 198, y: 59 }, + '2': { x: 35, y: 50 }, + '3': { x: 45, y: 50 }, + '4': { x: 25, y: 60 }, + '5': { x: 35, y: 60 }, + '6': { x: 45, y: 60 }, + '7': { x: 25, y: 70 }, + '8': { x: 35, y: 70 }, + '9': { x: 45, y: 70 }, + '0': { x: 35, y: 80 }, +}; + +/** Confirm button coordinate */ +export const CONFIRM_COORD = { x: 196, y: 68 }; + +/** Cancel/Back button coordinate */ +export const CANCEL_COORD = { x: 19, y: 87 }; + +/** Device button coordinates */ +export const DEVICE_BUTTONS = { + confirm: { x: 227, y: 94 }, + cancel: { x: 25, y: 85 }, + back: { x: 19, y: 87 }, + continue: { x: 212, y: 94 }, + next: { x: 212, y: 94 }, + finish: { x: 212, y: 94 }, +}; + +/** Arm home coordinate for Pro2. */ +export const DEVICE_HOME_COORD = { x: 0, y: 0 } as const; + +// ============================================================================ +// Helper functions +// ============================================================================ + +/** Delay (ms) after each letter tap when typing mnemonic words. */ +export const STEP_DELAY_AFTER_LETTER_MS = 700; +/** Delay (ms) after each word confirm tap to leave a small gap before the next word. */ +export const STEP_DELAY_AFTER_CONFIRM_MS = 1200; +/** Delay (ms) after the LAST word confirm — device needs time to validate the full mnemonic. */ +export const STEP_DELAY_AFTER_LAST_CONFIRM_MS = 5000; + +/** + * Generates AutoStep array from a list of words. + * Each word becomes: letter steps + confirm step. + */ +export function generateWordSteps(words: string[]): AutoStep[] { + const steps: AutoStep[] = []; + words.forEach((word, wordIndex) => { + const isLastWord = wordIndex === words.length - 1; + const lowerWord = word.toLowerCase(); + for (let i = 0; i < lowerWord.length; i++) { + const letter = lowerWord[i]; + const coord = LETTER_COORDS[letter]; + if (coord) { + steps.push({ + label: `W${wordIndex + 1}:${letter}`, + x: coord.x, + y: coord.y, + depth: 12, + delayAfter: STEP_DELAY_AFTER_LETTER_MS, + }); + } + } + steps.push({ + label: `W${wordIndex + 1}:confirm`, + x: CONFIRM_COORD.x, + y: CONFIRM_COORD.y, + depth: 12, + delayAfter: isLastWord ? STEP_DELAY_AFTER_LAST_CONFIRM_MS : STEP_DELAY_AFTER_CONFIRM_MS, + }); + }); + return steps; +} + +export function pickRandomShares(shares: string[][], count: number): string[][] { + const pool = [...shares]; + for (let i = pool.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [pool[i], pool[j]] = [pool[j], pool[i]]; + } + return pool.slice(0, Math.max(0, Math.min(count, pool.length))); +} + +export function generateSlip39ShareSteps(shares: string[][]): AutoStep[] { + const steps: AutoStep[] = []; + let globalWordIndex = 0; + + const isLastShare = (shareIndex: number) => shareIndex === shares.length - 1; + const isLastWordInShare = (wordInShareIndex: number, shareWords: string[]) => + wordInShareIndex === shareWords.length - 1; + + shares.forEach((shareWords, shareIndex) => { + shareWords.forEach((word, wordInShareIndex) => { + globalWordIndex += 1; + const isLastWord = isLastShare(shareIndex) && isLastWordInShare(wordInShareIndex, shareWords); + const lowerWord = word.toLowerCase(); + for (let i = 0; i < lowerWord.length; i++) { + const letter = lowerWord[i]; + const coord = LETTER_COORDS[letter]; + if (coord) { + steps.push({ + label: `W${globalWordIndex}:${letter}`, + x: coord.x, + y: coord.y, + depth: 12, + delayBefore: shareIndex > 0 && wordInShareIndex === 0 && i === 0 ? 3000 : undefined, + delayAfter: STEP_DELAY_AFTER_LETTER_MS, + }); + } + } + steps.push({ + label: `W${globalWordIndex}:confirm`, + x: CONFIRM_COORD.x, + y: CONFIRM_COORD.y, + depth: 12, + delayAfter: isLastWord ? STEP_DELAY_AFTER_LAST_CONFIRM_MS : STEP_DELAY_AFTER_CONFIRM_MS, + }); + }); + + if (shareIndex < shares.length - 1) { + steps.push({ + label: `Share${shareIndex + 1}:confirm`, + x: DEVICE_BUTTONS.confirm.x, + y: DEVICE_BUTTONS.confirm.y, + depth: 12, + delayBefore: 5000, + delayAfter: 5000, + }); + } + }); + + return steps; +} + +/** 12 words "all" input steps (legacy test) */ +const WORDS_12_STEPS: AutoStep[] = [ + // Word 1: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 2: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 3: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 4: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 5: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 6: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 7: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 8: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 9: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 10: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 11: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, + // Word 12: "all" + { label: '点击单词a', x: 194, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击单词l', x: 231, y: 82, depth: 12, delayAfter: 1000 }, + { label: '点击确认', x: 196, y: 68, depth: 12, delayAfter: 2000 }, +]; + +// ============================================================================ +// Page Action Definitions +// ============================================================================ + +/** All page actions organized by logical groups */ +const ALL_PAGE_ACTIONS: PageAction[] = [ + // -------------------------------------------------------------------------- + // 初始设置 (Initial Setup) + // -------------------------------------------------------------------------- + { + id: 'create-start-position', + name: '创建钱包起始坐标', + group: '初始设置', + steps: [ + { label: '移动到创建起始坐标', x: 180, y: 0, depth: 12, moveOnly: true, delayAfter: 500 }, + ], + }, + { + id: 'lang-zh', + name: '选择中文', + group: '初始设置', + steps: [ + { label: '点击Hello', x: 215, y: 65, depth: 12, delayAfter: 300 }, + { label: '引导选择语言1', x: 212, y: 66, depth: 12, delayAfter: 300 }, + { label: '引导选择语言2', x: 207, y: 73, depth: 12, delayAfter: 5000 }, + { label: '引导选择语言3', x: 213, y: 80, depth: 12, delayAfter: 300 }, + { label: '引导继续1', x: 202, y: 94, depth: 12, delayAfter: 300 }, + { label: '引导继续2', x: 202, y: 94, depth: 12, delayAfter: 300 }, + ], + }, + { + id: 'pin-1111', + name: '输入PIN码1111', + group: '初始设置', + steps: [ + { label: '输入PIN码1', x: 198, y: 59, depth: 12 }, + { label: '输入PIN码2', x: 198, y: 59, depth: 12 }, + { label: '输入PIN码3', x: 198, y: 59, depth: 12 }, + { label: '输入PIN码4', x: 198, y: 59, depth: 12 }, + { label: '点击确认', x: DEVICE_BUTTONS.confirm.x, y: DEVICE_BUTTONS.confirm.y, depth: 12 }, + { label: '再次确认PIN码1', x: 198, y: 59, depth: 12 }, + { label: '再次确认PIN码2', x: 198, y: 59, depth: 12 }, + { label: '再次确认PIN码3', x: 198, y: 59, depth: 12 }, + { label: '再次确认PIN码4', x: 198, y: 59, depth: 12 }, + { label: '点击确认', x: DEVICE_BUTTONS.confirm.x, y: DEVICE_BUTTONS.confirm.y, depth: 12, delayAfter: 900 }, + ], + }, + { + id: 'nav-continue-setup', + name: '继续+跳过指纹', + group: '初始设置', + steps: [ + { label: '提示页面继续', x: 212, y: 94, depth: 12, delayAfter: 500 }, + { label: '跳过指纹', x: 212, y: 94, depth: 12, delayAfter: 500 }, + { label: '跳过指纹确认', x: 212, y: 94, depth: 12, delayAfter: 500 }, + ], + }, + + // -------------------------------------------------------------------------- + // 钱包路径 (Wallet Path) + // -------------------------------------------------------------------------- + { + id: 'nav-import', + name: '导入钱包', + group: '钱包路径', + steps: [ + { label: '点击导入钱包', x: 212, y: 94, depth: 12 }, + ], + }, + { + id: 'nav-create', + name: '创建新钱包', + group: '钱包路径', + steps: [ + { label: '选择创建钱包', x: 212, y: 82, depth: 12, delayAfter: 500 }, + { label: '创建流程继续1', x: 212, y: 94, depth: 12, delayAfter: 500 }, + { label: '创建流程继续2', x: 212, y: 94, depth: 12, delayAfter: 500 }, + { label: '创建流程继续3', x: 212, y: 94, depth: 12, delayAfter: 1000 }, + ], + }, + + // -------------------------------------------------------------------------- + // 导入钱包 (Import Wallet) + // -------------------------------------------------------------------------- + { + id: 'select-mnemonic', + name: '选择助记词', + group: '导入钱包', + steps: [ + { label: '点击助记词', x: 212, y: 82, depth: 12 }, + ], + }, + + // -------------------------------------------------------------------------- + // 词数选择 (Word Count Selection) + // -------------------------------------------------------------------------- + { + id: 'select-12-words', + name: '选择12个单词', + group: '词数选择', + steps: [ + { label: '选择12个单词', x: 196, y: 61, depth: 12 }, + { label: '点击继续', x: 212, y: 94, depth: 12 }, + ], + }, + { + id: 'select-18-words', + name: '选择18个单词', + group: '词数选择', + steps: [ + { label: '选择18个单词', x: 196, y: 71, depth: 12 }, + { label: '点击继续', x: 212, y: 94, depth: 12 }, + ], + }, + { + id: 'select-20-words', + name: '选择20个单词', + group: '词数选择', + steps: [ + { label: '选择20个单词', x: 25, y: 70, depth: 12 }, + { label: '点击继续', x: 55, y: 85, depth: 12 }, + ], + }, + { + id: 'select-24-words', + name: '选择24个单词', + group: '词数选择', + steps: [ + { label: '选择24个单词', x: 196, y: 81, depth: 12 }, + { label: '点击继续', x: 212, y: 94, depth: 12 }, + ], + }, + { + id: 'select-33-words', + name: '选择33个单词', + group: '词数选择', + steps: [ + { label: '选择33个单词', x: 25, y: 90, depth: 12 }, + { label: '点击继续', x: 55, y: 85, depth: 12 }, + ], + }, + + // -------------------------------------------------------------------------- + // 创建钱包 (Create Wallet Flow) + // -------------------------------------------------------------------------- + { + id: 'create-backup-confirm', + name: '备份确认', + group: '创建钱包', + steps: [ + { label: '开始备份勾选1', x: 20, y: 40, depth: 12 }, + { label: '开始备份勾选2', x: 20, y: 50, depth: 12 }, + { label: '开始备份勾选3', x: 20, y: 65, depth: 12 }, + { label: '点击备份', x: 40, y: 85, depth: 12 }, + ], + }, + { + id: 'create-select-18-words', + name: '创建钱包选择18词', + group: '创建钱包', + steps: [ + { label: '展开助记词位数', x: 56, y: 23, depth: 12, delayAfter: 600 }, + { label: '选择18词', x: 40, y: 55, depth: 12, delayAfter: 800 }, + ], + }, + { + id: 'create-select-24-words', + name: '创建钱包选择24词', + group: '创建钱包', + steps: [ + { label: '展开助记词位数', x: 56, y: 23, depth: 12, delayAfter: 600 }, + { label: '选择24词', x: 40, y: 65, depth: 12, delayAfter: 800 }, + ], + }, + { + id: 'create-expand-word-options', + name: '创建钱包展开助记词选项', + group: '创建钱包', + steps: [ + { label: '展开助记词位数', x: 56, y: 23, depth: 12, delayAfter: 600 }, + ], + }, + { + id: 'create-mnemonic-scroll-10', + name: '助记词页上滑10', + group: '创建钱包', + steps: [ + { + label: '助记词页上滑10', + x: 50, + y: 78, + depth: 12, + swipeTo: { x: 50, y: 68 }, + swipeHoldDelay: 120, + delayAfter: 1200, + }, + ], + }, + { + id: 'create-mnemonic-scroll-15', + name: '助记词页上滑15', + group: '创建钱包', + steps: [ + { + label: '助记词页上滑15', + x: 50, + y: 78, + depth: 12, + swipeTo: { x: 50, y: 63 }, + swipeSegments: 5, + swipeSegmentDelay: 45, + swipeHoldDelay: 220, + delayAfter: 1400, + }, + ], + }, + { + id: 'create-slip39-mnemonic-scroll-15-slow', + name: 'SLIP39助记词页上滑15(慢速)', + group: '创建钱包', + steps: [ + { + // Use a larger, slower swipe so the second OCR pass reliably reveals words 9/10 and 19/20. + label: 'SLIP39助记词页上滑15(慢速)', + x: 50, + y: 78, + depth: 12, + swipeTo: { x: 50, y: 50 }, + swipeSegments: 12, + swipeSegmentDelay: 110, + swipeHoldDelay: 420, + delayAfter: 2600, + }, + ], + }, + { + id: 'create-mnemonic-scroll-20', + name: '助记词页上滑20', + group: '创建钱包', + steps: [ + { + // Keep page overlap while reducing inertial scroll before the second OCR capture. + label: '助记词页上滑20', + x: 50, + y: 78, + depth: 12, + swipeTo: { x: 50, y: 50 }, + swipeSegments: 10, + swipeSegmentDelay: 85, + swipeHoldDelay: 360, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-mnemonic-scroll-40', + name: '助记词页上滑40', + group: '创建钱包', + steps: [ + { + label: '助记词页上滑40', + x: 50, + y: 78, + depth: 12, + swipeTo: { x: 50, y: 38 }, + swipeHoldDelay: 240, + delayAfter: 1500, + }, + ], + }, + { + id: 'create-slip39-scroll-large', + name: 'SLIP39配置页大幅上滑', + group: '创建钱包', + steps: [ + { + label: 'SLIP39配置页大幅上滑', + x: 50, + y: 82, + depth: 12, + swipeTo: { x: 50, y: 38 }, + swipeHoldDelay: 240, + delayAfter: 1400, + }, + ], + }, + { + id: 'create-slip39-select-single', + name: '创建SLIP39选择单份', + group: '创建钱包', + steps: [ + { label: '选择单份助记词', x: 30, y: 45, depth: 12, delayAfter: 900 }, + ], + }, + { + id: 'create-slip39-select-multi', + name: '创建SLIP39选择多份', + group: '创建钱包', + steps: [ + { label: '选择多份助记词', x: 30, y: 55, depth: 12, delayAfter: 900 }, + ], + }, + { + id: 'create-slip39-shares-2', + name: 'SLIP39份额选择2', + group: '创建钱包', + steps: [ + { label: '选择2份额', x: 20, y: 35, depth: 12, delayAfter: 700 }, + ], + }, + { + id: 'create-slip39-shares-8', + name: 'SLIP39份额选择8', + group: '创建钱包', + steps: [ + { label: '选择8份额', x: 29, y: 44, depth: 12, delayAfter: 700 }, + ], + }, + { + id: 'create-slip39-shares-16', + name: 'SLIP39份额选择16', + group: '创建钱包', + steps: [ + { label: '选择16份额', x: 55, y: 53, depth: 12, delayAfter: 700 }, + ], + }, + { + id: 'create-slip39-threshold-2', + name: 'SLIP39阈值选择2', + group: '创建钱包', + steps: [ + { label: '选择2阈值', x: 20, y: 68, depth: 12, delayAfter: 700 }, + ], + }, + { + id: 'create-slip39-threshold-8', + name: 'SLIP39阈值选择8', + group: '创建钱包', + steps: [ + { label: '选择8阈值', x: 29, y: 77, depth: 12, delayAfter: 700 }, + ], + }, + { + id: 'create-slip39-config-continue', + name: 'SLIP39配置确认继续', + group: '创建钱包', + steps: [ + { label: '确认配置并继续', x: 55, y: 85, depth: 12, delayAfter: 900 }, + ], + }, + { + id: 'create-screenshot-12', + name: '截图识别(12词)', + group: '创建钱包', + steps: [ + { + label: '助记词页轻微上滑', + x: 212, + y: 80, + depth: 12, + swipeTo: { x: 212, y: 50 }, + swipeSegments: 1, + swipeHoldDelay: 120, + delayAfter: 1000, + }, + { + label: '移动到截图位置', + x: 259, + y: 0, + depth: 12, + ocrCapture: { + expectedWordCount: 12, + requireBip39: true, + sceneConfig: PRO2_OCR_SCENES.createWallet.mnemonic12, + }, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-screenshot-18', + name: '截图识别(18词)', + group: '创建钱包', + steps: [ + { + label: '移动到截图位置', + x: 85, + y: 0, + depth: 12, + ocrCapture: { expectedWordCount: 18, requireBip39: true }, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-screenshot-20', + name: '截图识别(20词)', + group: '创建钱包', + steps: [ + { + label: '移动到截图位置', + x: 85, + y: 0, + depth: 12, + ocrCapture: { expectedWordCount: 20, requireBip39: false }, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-screenshot-20-part1', + name: '截图识别(20词-第一页)', + group: '创建钱包', + steps: [ + { + label: '移动到截图位置(20词-1)', + x: 85, + y: 0, + depth: 12, + ocrCapture: { + expectedWordCount: 20, + mergeWithStored: true, + allowPartial: true, + requireBip39: false, + }, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-screenshot-20-part2', + name: '截图识别(20词-第二页)', + group: '创建钱包', + steps: [ + { + label: '移动到截图位置(20词-2)', + x: 85, + y: 0, + depth: 12, + ocrCapture: { + expectedWordCount: 20, + mergeWithStored: true, + allowPartial: false, + requireBip39: false, + }, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-screenshot-24-part1', + name: '截图识别(24词-第一页)', + group: '创建钱包', + steps: [ + { + label: '移动到截图位置(24词-1)', + x: 85, + y: 0, + depth: 12, + ocrCapture: { + expectedWordCount: 24, + mergeWithStored: true, + allowPartial: true, + requireBip39: false, + }, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-screenshot-24-part2', + name: '截图识别(24词-第二页)', + group: '创建钱包', + steps: [ + { + label: '移动到截图位置(24词-2)', + x: 85, + y: 0, + depth: 12, + ocrCapture: { + expectedWordCount: 24, + mergeWithStored: true, + allowPartial: false, + requireBip39: true, + }, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-continue', + name: '继续备份', + group: '创建钱包', + steps: [ + { label: '点击继续', x: 212, y: 94, depth: 12 }, + { label: '开始核对', x: 214, y: 82, depth: 12 }, + ], + }, + { + id: 'create-verify-word', + name: '验证单词', + group: '创建钱包', + steps: [ + { + label: '验证单词', + x: 259, y: 0, depth: 12, + ocrVerify: { + options: [ + { x: 213, y: 72, depth: 12 }, + { x: 213, y: 82, depth: 12 }, + { x: 213, y: 94, depth: 12 }, + ], + }, + delayAfter: 2000, + }, + ], + }, + { + id: 'create-final-continue-and-reset', + name: '确认后继续并复位', + group: '创建钱包', + steps: [ + { label: '点击继续1', x: 212, y: 94, depth: 12, delayAfter: 600 }, + { label: '点击继续2', x: 212, y: 94, depth: 12, delayAfter: 600 }, + { label: '点击继续3', x: 212, y: 94, depth: 12, delayAfter: 600 }, + { label: '点击继续4', x: 212, y: 94, depth: 12, delayAfter: 800 }, + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, + ], + }, + { + id: 'create-slip39-share-confirm', + name: 'SLIP39当前份额确认', + group: '创建钱包', + steps: [ + { label: '点击确认', x: 45, y: 85, depth: 12, delayBefore: 2000, delayAfter: 2000 }, + ], + }, + { + id: 'create-reset-only', + name: '仅复位', + group: '创建钱包', + steps: [ + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, + ], + }, + + + // -------------------------------------------------------------------------- + // 助记词输入 (Mnemonic Word Input) + // -------------------------------------------------------------------------- + { + id: 'input-words-12-all', + name: '12个词(all)', + group: '助记词输入', + steps: WORDS_12_STEPS, + }, + { + id: 'input-mnemonic-12-1', + name: '12词-1 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_12_1'], mode: 'single' }, + }, + { + id: 'input-mnemonic-12-2', + name: '12词-2 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_12_2'], mode: 'single' }, + }, + { + id: 'input-mnemonic-12-3', + name: '12词-3 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_12_3'], mode: 'single' }, + }, + { + id: 'input-mnemonic-12-api', + name: '签名方法 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_12_api'], mode: 'single' }, + }, + { + id: 'input-mnemonic-18-1', + name: '18词-1 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_18_1'], mode: 'single' }, + }, + { + id: 'input-mnemonic-18-2', + name: '18词-2 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_18_2'], mode: 'single' }, + }, + { + id: 'input-mnemonic-18-3', + name: '18词-3 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_18_3'], mode: 'single' }, + }, + { + id: 'input-mnemonic-24-1', + name: '24词-1 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_24_1'], mode: 'single' }, + }, + { + id: 'input-mnemonic-24-2', + name: '24词-2 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_24_2'], mode: 'single' }, + }, + { + id: 'input-mnemonic-24-3', + name: '24词-3 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'bip39', keys: ['mnemonic_24_3'], mode: 'single' }, + }, + { + id: 'input-slip39-20-1', + name: 'slip39-20词-1份 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'slip39', keys: ['slip39_20_1'], mode: 'single' }, + }, + { + id: 'input-slip39-20-2-all', + name: 'slip39-20词-2/3 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { + section: 'slip39', + keys: ['slip39_20_2_share1', 'slip39_20_2_share2', 'slip39_20_2_share3'], + mode: 'shares-random', + pickCount: 2, + }, + }, + { + id: 'input-slip39-20-16-all', + name: 'slip39-20词-16/16 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { + section: 'slip39', + keys: [ + 'slip39_20_16_share1', 'slip39_20_16_share2', 'slip39_20_16_share3', 'slip39_20_16_share4', + 'slip39_20_16_share5', 'slip39_20_16_share6', 'slip39_20_16_share7', 'slip39_20_16_share8', + 'slip39_20_16_share9', 'slip39_20_16_share10', 'slip39_20_16_share11', 'slip39_20_16_share12', + 'slip39_20_16_share13', 'slip39_20_16_share14', 'slip39_20_16_share15', 'slip39_20_16_share16', + ], + mode: 'shares-all', + }, + }, + { + id: 'input-slip39-33-1', + name: 'slip39-33词-1份 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { section: 'slip39', keys: ['slip39_33_1'], mode: 'single' }, + }, + { + id: 'input-slip39-33-2-all', + name: 'slip39-33词-3/2 输入', + group: '助记词输入', + steps: [], + mnemonicSource: { + section: 'slip39', + keys: ['slip39_33_2_share1', 'slip39_33_2_share2', 'slip39_33_2_share3'], + mode: 'shares-random', + pickCount: 2, + }, + }, + + // -------------------------------------------------------------------------- + // 完成 (Finish) + // -------------------------------------------------------------------------- + { + id: 'suffix-finish', + name: '完成流程', + group: '完成', + steps: [ + { label: '点击继续1', x: 212, y: 94, depth: 12 }, + { label: '点击继续2', x: 212, y: 94, depth: 12 }, + { label: '点击继续3', x: 212, y: 94, depth: 12 }, + { label: '点击继续4', x: 212, y: 94, depth: 12, delayAfter: 2000 }, + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, + ], + }, + { + id: 'suffix-finish-paced', + name: '完成流程(慢速)', + group: '完成', + steps: [ + { label: '点击继续1', x: 212, y: 94, depth: 12, delayAfter: 2000 }, + { label: '点击继续2', x: 212, y: 94, depth: 12, delayAfter: 2000 }, + { label: '点击继续3', x: 212, y: 94, depth: 12, delayAfter: 2000 }, + { label: '点击继续4', x: 212, y: 94, depth: 12, delayAfter: 3000 }, + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, + ], + }, + + // -------------------------------------------------------------------------- + // 设备管理 (Device Management) + // -------------------------------------------------------------------------- + { + id: 'reset-wallet-action', + name: '重置钱包流程', + group: '设备管理', + steps: [ + // Wake up password keyboard (single click is enough) + { label: 'Wake keyboard', x: DEVICE_BUTTONS.confirm.x, y: DEVICE_BUTTONS.confirm.y, depth: 12, delayAfter: 1000 }, + // Enter PIN 1111 + { label: 'PIN 1', x: NUMBER_COORDS['1'].x, y: NUMBER_COORDS['1'].y, depth: 12 }, + { label: 'PIN 2', x: NUMBER_COORDS['1'].x, y: NUMBER_COORDS['1'].y, depth: 12 }, + { label: 'PIN 3', x: NUMBER_COORDS['1'].x, y: NUMBER_COORDS['1'].y, depth: 12 }, + { label: 'PIN 4', x: NUMBER_COORDS['1'].x, y: NUMBER_COORDS['1'].y, depth: 12 }, + { label: 'Confirm', x: DEVICE_BUTTONS.confirm.x, y: DEVICE_BUTTONS.confirm.y, depth: 12, delayAfter: 2000 }, + // Enter reset flow + { label: '点击设置入口', x: 222, y: 75, depth: 12, delayAfter: 800 }, + { label: '点击钱包项', x: 202, y: 59, depth: 12, delayAfter: 800 }, + // Wallet 页面列表向上滑动一段距离后，点击重置按钮 + { + label: '滑动钱包列表', + x: 206, + y: 95, + depth: 12, + swipeTo: { x: 206, y: 35 }, + swipeSegments: 2, + swipeSegmentDelay: 40, + swipeHoldDelay: 300, + delayBefore: 500, + delayAfter: 1400, + }, + { label: '点击重置按钮', x: 206, y: 92, depth: 12, delayAfter: 600 }, + { label: '确认重置入口', x: 222, y: 67, depth: 12, delayAfter: 2000 }, + { label: '确认选项1', x: 196, y: 54, depth: 12, delayAfter: 800 }, + { label: '确认选项2', x: 196, y: 67, depth: 12, delayAfter: 900 }, + { + label: '滑动确认重置', + x: 196, + y: 91, + depth: 12, + swipeTo: { x: 231, y: 91 }, + swipeSegments: 2, + swipeSegmentDelay: 40, + swipeHoldDelay: 900, + delayAfter: 10000, + }, + // 复位机械臂：还原到 home 坐标 + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, + ], + }, + { + id: 'reset-wallet-unlocked-action', + name: '重置钱包流程(已解锁)', + group: '设备管理', + steps: [ + { label: 'Unlock swipe up', x: 50, y: 82, depth: 12, swipeTo: { x: 50, y: 38 }, swipeSegments: 8, swipeSegmentDelay: 80, swipeHoldDelay: 300, delayBefore: 500, delayAfter: 1800 }, + { label: 'Settings app', x: 50, y: 65, depth: 12, delayAfter: 600 }, + { label: 'Wallet section', x: 50, y: 55, depth: 12, delayAfter: 600 }, + { label: 'Swipe up', x: 35, y: 85, depth: 12, swipeTo: { x: 35, y: 70 }, delayAfter: 1000 }, + { label: 'Click 1', x: 50, y: 85, depth: 12, delayAfter: 800 }, + { label: 'Click 2', x: 50, y: 85, depth: 12, delayAfter: 800 }, + { label: 'Setting 1', x: 25, y: 44, depth: 12, delayAfter: 600 }, + { label: 'Setting 2', x: 25, y: 55, depth: 12, delayAfter: 900 }, + { + label: 'Swipe right', + x: 20, + y: 75, + depth: 12, + swipeTo: { x: 60, y: 75 }, + swipeSegments: 6, + swipeSegmentDelay: 70, + swipeHoldDelay: 900, + delayAfter: 7000, + }, + { label: 'Confirm', x: 25, y: 85, depth: 12, delayAfter: 10000 }, + { label: 'Reset position', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, + ], + }, +]; + +/** PageAction lookup map for O(1) access */ +const PAGE_ACTION_MAP = new Map( + ALL_PAGE_ACTIONS.map((a) => [a.id, a]) +); + +// ============================================================================ +// Sequence Definitions (composed from PageActions) +// ============================================================================ + +/** + * Auto sequence definition. + * Sequences are composed of ordered PageAction IDs. + */ +export interface AutoSequence { + id: string; + name: string; + category: string; + /** Ordered list of PageAction IDs that compose this sequence */ + actions: string[]; +} + +/** Import wallet shared action prefix */ +const IMPORT_PREFIX: string[] = [ + 'lang-zh', 'pin-1111', 'nav-continue-setup', 'nav-import', 'select-mnemonic', +]; + +/** Create wallet shared action prefix */ +const CREATE_PREFIX: string[] = [ + 'create-start-position', 'lang-zh', 'pin-1111', 'nav-continue-setup', 'nav-create', +]; +/** Create 18/24 wallet prefix: already on create page, do not click "创建新钱包". */ +const CREATE_PREFIX_DIRECT_EXPAND: string[] = [ + 'lang-zh', 'pin-1111', 'nav-continue-setup', +]; +const CREATE_FLOW_SUFFIX: string[] = [ + 'create-screenshot-12', + 'create-continue', + 'create-verify-word', + 'create-verify-word', + 'create-verify-word', + 'create-final-continue-and-reset', +]; +const CREATE_FLOW_SUFFIX_18: string[] = [ + 'create-backup-confirm', + 'create-mnemonic-scroll-10', + 'create-screenshot-18', + 'create-continue', + 'create-verify-word', + 'create-verify-word', + 'create-verify-word', + 'create-final-continue-and-reset', +]; +const CREATE_FLOW_SUFFIX_24: string[] = [ + 'create-backup-confirm', + 'create-screenshot-24-part1', + 'create-mnemonic-scroll-20', + 'create-screenshot-24-part2', + 'create-continue', + 'create-verify-word', + 'create-verify-word', + 'create-verify-word', + 'create-final-continue-and-reset', +]; +const CREATE_FLOW_SUFFIX_SLIP39_TEMPLATE: string[] = [ + 'create-backup-confirm', +]; +const CREATE_FLOW_SUFFIX_SLIP39_PER_SHARE: string[] = [ + 'create-screenshot-20-part1', + 'create-slip39-mnemonic-scroll-15-slow', + 'create-screenshot-20-part2', + 'create-continue', + 'create-verify-word', + 'create-verify-word', + 'create-verify-word', + 'create-slip39-share-confirm', +]; + +function buildSlip39PerShareActions(shareCount: number): string[] { + const loops = Math.max(1, Math.floor(shareCount)); + const actions: string[] = []; + for (let i = 0; i < loops; i++) { + actions.push(...CREATE_FLOW_SUFFIX_SLIP39_PER_SHARE); + } + return actions; +} + +const ALL_SEQUENCES: AutoSequence[] = [ + // ============================================================================ + // 设备管理 + // ============================================================================ + { + id: 'reset-wallet', + name: '重置钱包', + category: '设备管理', + actions: ['reset-wallet-action'], + }, + { + id: 'reset-wallet-locked', + name: '重置钱包(锁定态)', + category: '设备管理', + actions: ['reset-wallet-action'], + }, + { + id: 'reset-wallet-unlocked', + name: '重置钱包(已解锁)', + category: '设备管理', + actions: ['reset-wallet-unlocked-action'], + }, + + // ============================================================================ + // 创建钱包 + // ============================================================================ + { + id: 'create-wallet', + name: '创建新钱包(12词)', + category: '创建钱包', + actions: [...CREATE_PREFIX, ...CREATE_FLOW_SUFFIX], + }, + { + id: 'create-wallet-18', + name: '创建新钱包(18词)', + category: '创建钱包', + actions: [...CREATE_PREFIX_DIRECT_EXPAND, 'create-select-18-words', ...CREATE_FLOW_SUFFIX_18], + }, + { + id: 'create-wallet-24', + name: '创建新钱包(24词)', + category: '创建钱包', + actions: [...CREATE_PREFIX_DIRECT_EXPAND, 'create-select-24-words', ...CREATE_FLOW_SUFFIX_24], + }, + { + id: 'create-slip39-single-template', + name: '创建SLIP39(单份模板)', + category: '创建钱包', + actions: [ + ...CREATE_PREFIX_DIRECT_EXPAND, + 'create-expand-word-options', + 'create-slip39-scroll-large', + 'create-slip39-select-single', + ...CREATE_FLOW_SUFFIX_SLIP39_TEMPLATE, + ...buildSlip39PerShareActions(1), + 'create-final-continue-and-reset', + ], + }, + { + id: 'create-slip39-multi-2of2-template', + name: '创建SLIP39(多份模板 2/2)', + category: '创建钱包', + actions: [ + ...CREATE_PREFIX_DIRECT_EXPAND, + 'create-expand-word-options', + 'create-slip39-scroll-large', + 'create-slip39-select-multi', + 'create-slip39-shares-2', + 'create-slip39-threshold-2', + 'create-slip39-config-continue', + ...CREATE_FLOW_SUFFIX_SLIP39_TEMPLATE, + ...buildSlip39PerShareActions(2), + 'create-final-continue-and-reset', + ], + }, + { + id: 'create-slip39-multi-8of8-template', + name: '创建SLIP39(多份模板 8/8)', + category: '创建钱包', + actions: [ + ...CREATE_PREFIX_DIRECT_EXPAND, + 'create-expand-word-options', + 'create-slip39-scroll-large', + 'create-slip39-select-multi', + 'create-slip39-shares-8', + 'create-slip39-threshold-8', + 'create-slip39-config-continue', + ...CREATE_FLOW_SUFFIX_SLIP39_TEMPLATE, + ...buildSlip39PerShareActions(8), + 'create-final-continue-and-reset', + ], + }, + { + id: 'create-slip39-multi-16of2-template', + name: '创建SLIP39(多份模板 16/2)', + category: '创建钱包', + actions: [ + ...CREATE_PREFIX_DIRECT_EXPAND, + 'create-expand-word-options', + 'create-slip39-scroll-large', + 'create-slip39-select-multi', + 'create-slip39-shares-16', + 'create-slip39-threshold-2', + 'create-slip39-config-continue', + ...CREATE_FLOW_SUFFIX_SLIP39_TEMPLATE, + ...buildSlip39PerShareActions(16), + 'create-final-continue-and-reset', + ], + }, + + // ============================================================================ + // BIP39 12词 + // ============================================================================ + { + id: 'words-12', + name: '12个词(all)', + category: 'BIP39 12词', + actions: [...IMPORT_PREFIX, 'select-12-words', 'input-words-12-all', 'suffix-finish-paced'], + }, + { + id: 'one-normal-12', + name: '12词-1', + category: 'BIP39 12词', + actions: [...IMPORT_PREFIX, 'select-12-words', 'input-mnemonic-12-1', 'suffix-finish-paced'], + }, + { + id: 'two-normal-12', + name: '12词-2', + category: 'BIP39 12词', + actions: [...IMPORT_PREFIX, 'select-12-words', 'input-mnemonic-12-2', 'suffix-finish-paced'], + }, + { + id: 'three-normal-12', + name: '12词-3', + category: 'BIP39 12词', + actions: [...IMPORT_PREFIX, 'select-12-words', 'input-mnemonic-12-3', 'suffix-finish-paced'], + }, + { + id: 'api-normal-12', + name: '签名方法', + category: 'BIP39 12词', + actions: [...IMPORT_PREFIX, 'select-12-words', 'input-mnemonic-12-api', 'suffix-finish-paced'], + }, + + // ============================================================================ + // BIP39 18词 + // ============================================================================ + { + id: 'one-normal-18', + name: '18词-1', + category: 'BIP39 18词', + actions: [...IMPORT_PREFIX, 'select-18-words', 'input-mnemonic-18-1', 'suffix-finish-paced'], + }, + { + id: 'two-normal-18', + name: '18词-2', + category: 'BIP39 18词', + actions: [...IMPORT_PREFIX, 'select-18-words', 'input-mnemonic-18-2', 'suffix-finish-paced'], + }, + { + id: 'three-normal-18', + name: '18词-3', + category: 'BIP39 18词', + actions: [...IMPORT_PREFIX, 'select-18-words', 'input-mnemonic-18-3', 'suffix-finish-paced'], + }, + + // ============================================================================ + // BIP39 24词 + // ============================================================================ + { + id: 'one-normal-24', + name: '24词-1', + category: 'BIP39 24词', + actions: [...IMPORT_PREFIX, 'select-24-words', 'input-mnemonic-24-1', 'suffix-finish-paced'], + }, + { + id: 'two-normal-24', + name: '24词-2', + category: 'BIP39 24词', + actions: [...IMPORT_PREFIX, 'select-24-words', 'input-mnemonic-24-2', 'suffix-finish-paced'], + }, + { + id: 'three-normal-24', + name: '24词-3', + category: 'BIP39 24词', + actions: [...IMPORT_PREFIX, 'select-24-words', 'input-mnemonic-24-3', 'suffix-finish-paced'], + }, + + // ============================================================================ + // SLIP39 20词 + // ============================================================================ + { + id: 'count20_one_normal', + name: 'slip39-20词-1份', + category: 'SLIP39 20词', + actions: [...IMPORT_PREFIX, 'select-20-words', 'input-slip39-20-1', 'suffix-finish-paced'], + }, + { + id: 'count20_two_normal', + name: 'slip39-20词-2/3', + category: 'SLIP39 20词', + actions: [...IMPORT_PREFIX, 'select-20-words', 'input-slip39-20-2-all', 'suffix-finish-paced'], + }, + { + id: 'count20_three_normal', + name: 'slip39-20词-16/16', + category: 'SLIP39 20词', + actions: [...IMPORT_PREFIX, 'select-20-words', 'input-slip39-20-16-all', 'suffix-finish-paced'], + }, + + // ============================================================================ + // SLIP39 33词 + // ============================================================================ + { + id: 'count33_one_normal', + name: 'slip39-33词-1份', + category: 'SLIP39 33词', + actions: [...IMPORT_PREFIX, 'select-33-words', 'input-slip39-33-1', 'suffix-finish-paced'], + }, + { + id: 'count33_two_normal', + name: 'slip39-33词-3/2', + category: 'SLIP39 33词', + actions: [...IMPORT_PREFIX, 'select-33-words', 'input-slip39-33-2-all', 'suffix-finish-paced'], + }, +]; + +const SUPPORTED_PRO2_SEQUENCE_IDS = new Set([ + 'reset-wallet', + 'reset-wallet-locked', + 'create-wallet', + 'words-12', + 'one-normal-12', + 'two-normal-12', + 'three-normal-12', + 'api-normal-12', + 'one-normal-18', + 'two-normal-18', + 'three-normal-18', + 'one-normal-24', + 'two-normal-24', + 'three-normal-24', +]); + +const VISIBLE_SEQUENCES: AutoSequence[] = ALL_SEQUENCES.filter((sequence) => + SUPPORTED_PRO2_SEQUENCE_IDS.has(sequence.id) +); + +// ============================================================================ +// PageAction Query Functions +// ============================================================================ + +/** + * Gets a page action by ID. + */ +export function getPageAction(id: string): PageAction | undefined { + return PAGE_ACTION_MAP.get(id); +} + +/** + * Gets all page actions. + */ +export function getAllPageActions(): PageAction[] { + return [...ALL_PAGE_ACTIONS]; +} + +/** + * Gets page actions filtered by group. + */ +export function getPageActionsByGroup(group: string): PageAction[] { + return ALL_PAGE_ACTIONS.filter((a) => a.group === group); +} + +/** + * Gets all unique page action groups in display order. + */ +export function getAllPageActionGroups(): string[] { + const seen = new Set(); + const groups: string[] = []; + for (const action of ALL_PAGE_ACTIONS) { + if (!seen.has(action.group)) { + seen.add(action.group); + groups.push(action.group); + } + } + return groups; +} + +// ============================================================================ +// Sequence Query Functions +// ============================================================================ + +/** + * Gets a sequence by ID. + */ +export function getSequence(id: string): AutoSequence | undefined { + return VISIBLE_SEQUENCES.find((s) => s.id === id); +} + +/** + * Gets all available sequence IDs. + */ +export function getAllSequenceIds(): string[] { + return VISIBLE_SEQUENCES.map((s) => s.id); +} + +/** + * Gets all unique categories in display order. + */ +export function getAllCategories(): string[] { + const seen = new Set(); + const categories: string[] = []; + for (const seq of VISIBLE_SEQUENCES) { + if (!seen.has(seq.category)) { + seen.add(seq.category); + categories.push(seq.category); + } + } + return categories; +} + +/** + * Gets sequences filtered by category. + */ +export function getSequencesByCategory(category: string): AutoSequence[] { + return VISIBLE_SEQUENCES.filter((s) => s.category === category); +} + +/** + * Resolves a sequence's actions into a flat list of AutoSteps. + * Each action ID is looked up in the PageAction registry and its steps are concatenated. + */ +export function getFullSteps(sequence: AutoSequence): AutoStep[] { + const steps: AutoStep[] = []; + for (const actionId of sequence.actions) { + const action = PAGE_ACTION_MAP.get(actionId); + if (action) { + steps.push(...(action.buildSteps ? action.buildSteps() : action.steps)); + } else { + console.warn(`[sequences] Unknown page action ID: ${actionId}`); + } + } + return steps; +} diff --git a/electron/mcp/sequenceSetResolver.ts b/electron/mcp/sequenceSetResolver.ts new file mode 100644 index 0000000..5631047 --- /dev/null +++ b/electron/mcp/sequenceSetResolver.ts @@ -0,0 +1,42 @@ +import { + resolvePageActionSteps, + resolveSequenceSteps, + resolveSequenceStepsById, +} from './sequenceResolver'; +import { + resolvePro2PageActionSteps, + resolvePro2SequenceSteps, + resolvePro2SequenceStepsById, +} from './pro2SequenceResolver'; +import { normalizeDeviceTestSetId, type DeviceTestSetId } from './sequenceSets'; + +import type { AutoSequence, AutoStep, PageAction } from './sequences'; + +export function resolvePageActionStepsForDevice( + action: PageAction, + deviceTestSetId?: string | null +): AutoStep[] { + return normalizeDeviceTestSetId(deviceTestSetId) === 'pro2' + ? resolvePro2PageActionSteps(action as never) as AutoStep[] + : resolvePageActionSteps(action); +} + +export function resolveSequenceStepsForDevice( + sequence: AutoSequence, + deviceTestSetId?: string | null +): AutoStep[] { + return normalizeDeviceTestSetId(deviceTestSetId) === 'pro2' + ? resolvePro2SequenceSteps(sequence as never) as AutoStep[] + : resolveSequenceSteps(sequence); +} + +export function resolveSequenceStepsByIdForDevice( + sequenceId: string, + deviceTestSetId?: string | null +): AutoStep[] { + return normalizeDeviceTestSetId(deviceTestSetId) === 'pro2' + ? resolvePro2SequenceStepsById(sequenceId) as AutoStep[] + : resolveSequenceStepsById(sequenceId); +} + +export type { DeviceTestSetId }; diff --git a/electron/mcp/sequenceSets.ts b/electron/mcp/sequenceSets.ts new file mode 100644 index 0000000..e29f646 --- /dev/null +++ b/electron/mcp/sequenceSets.ts @@ -0,0 +1,60 @@ +import * as proSequences from './sequences'; +import * as pro2Sequences from './pro2Sequences'; + +import type { AutoSequence, PageAction } from './sequences'; + +export type DeviceTestSetId = 'pro' | 'pro2'; + +export interface DeviceTestSetOption { + id: DeviceTestSetId; + name: string; +} + +export const DEFAULT_DEVICE_TEST_SET_ID: DeviceTestSetId = 'pro'; + +export const DEVICE_TEST_SETS: DeviceTestSetOption[] = [ + { id: 'pro', name: 'Pro' }, + { id: 'pro2', name: 'Pro2' }, +]; + +export function normalizeDeviceTestSetId(id?: string | null): DeviceTestSetId { + return id === 'pro2' ? 'pro2' : DEFAULT_DEVICE_TEST_SET_ID; +} + +function getSequenceModule(deviceTestSetId?: string | null) { + return normalizeDeviceTestSetId(deviceTestSetId) === 'pro2' ? pro2Sequences : proSequences; +} + +export function getSequence(id: string, deviceTestSetId?: string | null): AutoSequence | undefined { + return getSequenceModule(deviceTestSetId).getSequence(id) as AutoSequence | undefined; +} + +export function getPageAction(id: string, deviceTestSetId?: string | null): PageAction | undefined { + return getSequenceModule(deviceTestSetId).getPageAction(id) as PageAction | undefined; +} + +export function getAllSequenceIds(deviceTestSetId?: string | null): string[] { + return getSequenceModule(deviceTestSetId).getAllSequenceIds(); +} + +export function getAllCategories(deviceTestSetId?: string | null): string[] { + return getSequenceModule(deviceTestSetId).getAllCategories(); +} + +export function getSequencesByCategory( + category: string, + deviceTestSetId?: string | null +): AutoSequence[] { + return getSequenceModule(deviceTestSetId).getSequencesByCategory(category) as AutoSequence[]; +} + +export function getFullSteps( + sequence: AutoSequence, + deviceTestSetId?: string | null +): proSequences.AutoStep[] { + return getSequenceModule(deviceTestSetId).getFullSteps(sequence as never) as proSequences.AutoStep[]; +} + +export function getDeviceHomeCoord(deviceTestSetId?: string | null): { x: number; y: number } { + return getSequenceModule(deviceTestSetId).DEVICE_HOME_COORD; +} diff --git a/electron/mcp/sequences.ts b/electron/mcp/sequences.ts index 2dc5bee..07f9a00 100644 --- a/electron/mcp/sequences.ts +++ b/electron/mcp/sequences.ts @@ -7,6 +7,7 @@ * * These sequences are used by both MCP tools and UI. */ +import type { SequenceOcrSceneConfig } from './ocrScenes'; /** Represents a single step in the auto operation sequence */ export interface OcrCaptureOptions { @@ -18,6 +19,8 @@ export interface OcrCaptureOptions { allowPartial?: boolean; /** Whether checksum validation is required for this capture to be treated as success. */ requireBip39?: boolean; + /** Optional OCR crop override for device-specific camera framing. */ + sceneConfig?: SequenceOcrSceneConfig; } export interface AutoStep { @@ -37,6 +40,8 @@ export interface AutoStep { swipeSegmentDelay?: number; /** Delay in ms before raising stylus after swipe (default: 50ms) */ swipeHoldDelay?: number; + /** If set, moves arm to position without clicking. */ + moveOnly?: boolean; /** If set, moves arm to position without clicking, then triggers OCR capture. */ ocrCapture?: boolean | OcrCaptureOptions; /** If set, performs verification OCR and clicks the correct option */ @@ -117,6 +122,9 @@ export const DEVICE_BUTTONS = { finish: { x: 55, y: 85 }, }; +/** Arm home coordinate for Pro. */ +export const DEVICE_HOME_COORD = { x: 0, y: 0 } as const; + // ============================================================================ // Helper functions // ============================================================================ @@ -781,7 +789,7 @@ const ALL_PAGE_ACTIONS: PageAction[] = [ { label: '点击继续2', x: 45, y: 85, depth: 12, delayAfter: 600 }, { label: '点击继续3', x: 45, y: 85, depth: 12, delayAfter: 600 }, { label: '点击继续4', x: 45, y: 85, depth: 12, delayAfter: 800 }, - { label: '复位', x: 0, y: 0, depth: 12 }, + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, ], }, { @@ -797,7 +805,7 @@ const ALL_PAGE_ACTIONS: PageAction[] = [ name: '仅复位', group: '创建钱包', steps: [ - { label: '复位', x: 0, y: 0, depth: 12 }, + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, ], }, @@ -947,7 +955,7 @@ const ALL_PAGE_ACTIONS: PageAction[] = [ { label: '点击继续', x: 55, y: 85, depth: 12 }, { label: '点击下一步', x: 55, y: 85, depth: 12 }, { label: '点击完成', x: 55, y: 85, depth: 12, delayAfter: 2000 }, - { label: '复位', x: 0, y: 0, depth: 12 }, + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, ], }, { @@ -958,7 +966,7 @@ const ALL_PAGE_ACTIONS: PageAction[] = [ { label: '点击继续', x: 55, y: 85, depth: 12, delayAfter: 2000 }, { label: '点击下一步', x: 55, y: 85, depth: 12, delayAfter: 2000 }, { label: '点击完成', x: 55, y: 85, depth: 12, delayAfter: 3000 }, - { label: '复位', x: 0, y: 0, depth: 12 }, + { label: '复位', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, ], }, @@ -1004,7 +1012,7 @@ const ALL_PAGE_ACTIONS: PageAction[] = [ // Final confirmation with wait { label: 'Confirm', x: 25, y: 85, depth: 12, delayAfter: 10000 }, // Reset to origin - { label: 'Reset position', x: 0, y: 0, depth: 12 }, + { label: 'Reset position', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, ], }, { @@ -1032,7 +1040,7 @@ const ALL_PAGE_ACTIONS: PageAction[] = [ delayAfter: 7000, }, { label: 'Confirm', x: 25, y: 85, depth: 12, delayAfter: 10000 }, - { label: 'Reset position', x: 0, y: 0, depth: 12 }, + { label: 'Reset position', x: DEVICE_HOME_COORD.x, y: DEVICE_HOME_COORD.y, depth: 12 }, ], }, ]; diff --git a/electron/mcp/state.ts b/electron/mcp/state.ts index 6ab9456..cc95a39 100644 --- a/electron/mcp/state.ts +++ b/electron/mcp/state.ts @@ -2,6 +2,7 @@ * Shared state management for MCP Server. * Maintains arm connection status, position, and provides state accessors. */ +import type { SequenceOcrSceneConfig } from './ocrScenes'; export interface ArmState { /** Whether the arm is connected */ @@ -68,6 +69,7 @@ export interface MnemonicOcrRequest { mergeWithStored?: boolean; allowPartial?: boolean; requireBip39?: boolean; + sceneConfig?: SequenceOcrSceneConfig; } export interface MnemonicStoreMetadata { @@ -119,7 +121,8 @@ export interface RendererSequenceExecutionResult { type MnemonicOcrCallback = (request?: MnemonicOcrRequest) => Promise; type VerifyOcrCallback = () => Promise; type RendererSequenceExecutionCallback = ( - sequenceId: string + sequenceId: string, + deviceTestSetId?: string ) => Promise; /** Frame capture function (set by main process when renderer is ready) */ @@ -253,13 +256,14 @@ export function setRendererSequenceExecutionCallback( } export async function runSequenceInRenderer( - sequenceId: string + sequenceId: string, + deviceTestSetId?: string ): Promise { if (!rendererSequenceExecutionCallback) { console.warn('Renderer sequence execution callback not set'); return null; } - return rendererSequenceExecutionCallback(sequenceId); + return rendererSequenceExecutionCallback(sequenceId, deviceTestSetId); } /** diff --git a/electron/mcp/tools/executeSequence.ts b/electron/mcp/tools/executeSequence.ts index 473d39e..ae24fed 100644 --- a/electron/mcp/tools/executeSequence.ts +++ b/electron/mcp/tools/executeSequence.ts @@ -22,8 +22,14 @@ import { setStopSequenceFlag, runSequenceInRenderer, } from '../state'; -import { getSequence, getPageAction, getAllSequenceIds } from '../sequences'; -import { resolvePageActionSteps } from '../sequenceResolver'; +import { + DEFAULT_DEVICE_TEST_SET_ID, + getAllSequenceIds, + getPageAction, + getSequence, + normalizeDeviceTestSetId, +} from '../sequenceSets'; +import { resolvePageActionStepsForDevice } from '../sequenceSetResolver'; import { saveCaptureToDownloads } from '../../saveCapture'; import { executeClickStep, executeSwipeStep } from '../utils/executeStep'; @@ -33,7 +39,12 @@ import type { AutoStep } from '../sequences'; export const executeSequenceSchema = z.object({ sequenceId: z .string() - .describe(`The ID of the sequence to execute. Available: ${getAllSequenceIds().join(', ')}`), + .describe(`The ID of the sequence to execute. Available for each device set: ${getAllSequenceIds().join(', ')}`), + deviceTestSetId: z + .enum(['pro', 'pro2']) + .optional() + .default(DEFAULT_DEVICE_TEST_SET_ID) + .describe('Device test set to execute. Pro2 has a separate sequence definition file.'), returnFrame: z .boolean() .optional() @@ -49,6 +60,7 @@ export interface ExecuteSequenceOutput { message: string; sequenceId?: string; sequenceName?: string; + deviceTestSetId?: string; stepsCompleted?: number; totalSteps?: number; } @@ -90,7 +102,10 @@ async function executeStep( }; const stepConfig = { clickDelay: ARM_CONFIG.clickDelay, zUp: ARM_CONFIG.zUp }; - if (step.ocrCapture) { + if (step.moveOnly) { + await send(`X${step.x}Y${step.y}`); + updateArmState({ currentX: step.x, currentY: step.y }); + } else if (step.ocrCapture) { const ocrCaptureConfig = typeof step.ocrCapture === 'object' ? step.ocrCapture : {}; // OCR capture step: move arm out of the way without clicking await send(`X${step.x}Y${step.y}`); @@ -192,23 +207,26 @@ export async function executeExecuteSequence( httpRequest: (url: string) => Promise ): Promise<{ output: ExecuteSequenceOutput; frame: string | null }> { const state = getArmState(); + const deviceTestSetId = normalizeDeviceTestSetId(input.deviceTestSetId); if (!state.isConnected || state.resourceHandle <= 0) { return { output: { success: false, message: 'Not connected to arm controller. Call arm-connect first.', + deviceTestSetId, }, frame: null, }; } - const sequence = getSequence(input.sequenceId); + const sequence = getSequence(input.sequenceId, deviceTestSetId); if (!sequence) { return { output: { success: false, - message: `Unknown sequence ID: ${input.sequenceId}. Available: ${getAllSequenceIds().join(', ')}`, + message: `Unknown ${deviceTestSetId} sequence ID: ${input.sequenceId}. Available: ${getAllSequenceIds(deviceTestSetId).join(', ')}`, + deviceTestSetId, }, frame: null, }; @@ -219,19 +237,20 @@ export async function executeExecuteSequence( // 2. Actions with buildSteps (e.g. random share selection) are only evaluated once const resolvedActions: Array<{ action: NonNullable>; steps: AutoStep[] }> = []; for (const actionId of sequence.actions) { - const action = getPageAction(actionId); + const action = getPageAction(actionId, deviceTestSetId); if (!action) { return { output: { success: false, - message: `Unknown page action ID: ${actionId}`, + message: `Unknown ${deviceTestSetId} page action ID: ${actionId}`, sequenceId: sequence.id, sequenceName: sequence.name, + deviceTestSetId, }, frame: null, }; } - resolvedActions.push({ action, steps: resolvePageActionSteps(action) }); + resolvedActions.push({ action, steps: resolvePageActionStepsForDevice(action, deviceTestSetId) }); } const totalSteps = resolvedActions.reduce((sum, { steps }) => sum + steps.length, 0); let stepsCompleted = 0; @@ -242,7 +261,7 @@ export async function executeExecuteSequence( setStopSequenceFlag(false); clearMnemonicWords(); - const rendererResult = await runSequenceInRenderer(sequence.id); + const rendererResult = await runSequenceInRenderer(sequence.id, deviceTestSetId); if (rendererResult) { if (rendererResult.mnemonicState?.words?.length) { storeStructuredMnemonicState( @@ -280,6 +299,7 @@ export async function executeExecuteSequence( : rendererResult.message, sequenceId: sequence.id, sequenceName: sequence.name, + deviceTestSetId, stepsCompleted: rendererResult.stepsCompleted, totalSteps: rendererResult.totalSteps, }, @@ -299,6 +319,7 @@ export async function executeExecuteSequence( message: `Sequence "${sequence.name}" stopped by user at step ${stepsCompleted + 1}`, sequenceId: sequence.id, sequenceName: sequence.name, + deviceTestSetId, stepsCompleted, totalSteps, }, @@ -374,6 +395,7 @@ export async function executeExecuteSequence( : `Sequence "${sequence.name}" completed successfully`, sequenceId: sequence.id, sequenceName: sequence.name, + deviceTestSetId, stepsCompleted, totalSteps, }, @@ -387,6 +409,7 @@ export async function executeExecuteSequence( message: `Sequence execution failed at step ${stepsCompleted + 1}: ${errorMessage}`, sequenceId: sequence.id, sequenceName: sequence.name, + deviceTestSetId, stepsCompleted, totalSteps, }, diff --git a/electron/paddleOcrEn.ts b/electron/paddleOcrEn.ts index c3a11f7..57641c6 100644 --- a/electron/paddleOcrEn.ts +++ b/electron/paddleOcrEn.ts @@ -8,6 +8,7 @@ export interface PaddleOcrEnRequest { imageDataUrl: string; layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic'; expectedWordCount?: number; + recVariantCount?: number; } export interface PaddleOcrEnResult { diff --git a/electron/preload.ts b/electron/preload.ts index b65d839..d1086f2 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -15,10 +15,16 @@ interface MnemonicOcrRequestPayload { mergeWithStored?: boolean; allowPartial?: boolean; requireBip39?: boolean; + sceneConfig?: { + roi: { x: number; y: number; width: number; height: number }; + scale?: number; + useNearestNeighbor?: boolean; + }; } interface RendererSequenceExecutionRequestPayload { sequenceId: string; + deviceTestSetId?: string; armState: { isConnected: boolean; resourceHandle: number; @@ -34,6 +40,7 @@ interface RendererSequenceExecutionResponsePayload { message: string; sequenceId: string; sequenceName?: string; + deviceTestSetId?: string; stepsCompleted: number; totalSteps: number; mnemonicState?: { @@ -75,6 +82,7 @@ interface ResolvedSequenceStepPayload { swipeSegments?: number; swipeSegmentDelay?: number; swipeHoldDelay?: number; + moveOnly?: boolean; ocrCapture?: boolean | { expectedWordCount?: number; mergeWithStored?: boolean; @@ -112,8 +120,8 @@ contextBridge.exposeInMainWorld('electronAPI', { // HTTP request (bypasses CORS by going through main process) httpRequest: (url: string) => ipcRenderer.invoke('http-request', url), - resolveSequenceSteps: (sequenceId: string) => - ipcRenderer.invoke('resolve-sequence-steps', sequenceId) as Promise, + resolveSequenceSteps: (sequenceId: string, deviceTestSetId?: string) => + ipcRenderer.invoke('resolve-sequence-steps', sequenceId, deviceTestSetId) as Promise, tryRecoverArmConnection: (payload: { serverIP: string; comPort: string }) => ipcRenderer.invoke('try-recover-arm-connection', payload) as Promise<{ @@ -228,12 +236,14 @@ contextBridge.exposeInMainWorld('electronAPI', { paddleOcrEnRecognize: ( imageDataUrl: string, layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic', - expectedWordCount?: number + expectedWordCount?: number, + recVariantCount?: number ) => ipcRenderer.invoke('paddleocr-en-recognize', { imageDataUrl, layoutHint, expectedWordCount, + recVariantCount, }) as Promise, // MCP Log: Receive log entries from main process @@ -271,7 +281,7 @@ declare global { onMainProcessMessage: (callback: (message: string) => void) => void; sendMessage: (channel: string, data: unknown) => void; httpRequest: (url: string) => Promise<{ status: number; data: string }>; - resolveSequenceSteps: (sequenceId: string) => Promise; + resolveSequenceSteps: (sequenceId: string, deviceTestSetId?: string) => Promise; tryRecoverArmConnection: (payload: { serverIP: string; comPort: string; @@ -316,7 +326,8 @@ declare global { paddleOcrEnRecognize: ( imageDataUrl: string, layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic', - expectedWordCount?: number + expectedWordCount?: number, + recVariantCount?: number ) => Promise; onMcpServerReady: (callback: (info: { port: number }) => void) => void; // MCP Logs diff --git a/scripts/paddleocr_en_infer.py b/scripts/paddleocr_en_infer.py index cc9829f..9c3b33d 100644 --- a/scripts/paddleocr_en_infer.py +++ b/scripts/paddleocr_en_infer.py @@ -79,6 +79,12 @@ def read_int_env(name: str, default: int, minimum: int, maximum: int) -> int: minimum=1, maximum=32, ) +REC_VARIANT_COUNT = read_int_env( + "QA_AUTO_HW_OCR_REC_VARIANTS", + default=1, + minimum=1, + maximum=4, +) try: @@ -257,6 +263,52 @@ def preprocess_rec_input(crop_bgr: np.ndarray) -> np.ndarray: return canvas.transpose(2, 0, 1)[None, :] +def build_rec_crop_variants(crop_bgr: np.ndarray, variant_count: int = REC_VARIANT_COUNT) -> List[np.ndarray]: + variants = [crop_bgr] + safe_variant_count = max(1, min(4, int(variant_count))) + if safe_variant_count <= 1 or crop_bgr.size == 0: + return variants + + # Light sharpening helps cyan/white OLED text that is slightly bloomed by the camera. + blurred = cv2.GaussianBlur(crop_bgr, (0, 0), 1.0) + sharpened = cv2.addWeighted(crop_bgr, 1.55, blurred, -0.55, 0) + variants.append(sharpened) + if safe_variant_count <= 2: + return variants + + # Local contrast on luminance keeps text strokes stronger without changing geometry. + lab = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2LAB) + l_channel, a_channel, b_channel = cv2.split(lab) + clahe = cv2.createCLAHE(clipLimit=2.2, tileGridSize=(4, 4)) + contrast_l = clahe.apply(l_channel) + contrast = cv2.cvtColor(cv2.merge((contrast_l, a_channel, b_channel)), cv2.COLOR_LAB2BGR) + variants.append(contrast) + if safe_variant_count <= 3: + return variants + + # High-contrast binary fallback for very clear bright text on dark screen. + gray = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2GRAY) + _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + variants.append(cv2.cvtColor(binary, cv2.COLOR_GRAY2BGR)) + return variants + + +def score_rec_text(text: str, conf: float) -> float: + token = extract_alpha_token(text) + score = conf + if 3 <= len(token) <= 8: + score += 0.08 + elif token: + score -= 0.08 + else: + score -= 0.25 + + alpha_tokens = re.findall(r"[A-Za-z]+", text) + if len(alpha_tokens) > 1: + score -= 0.04 * (len(alpha_tokens) - 1) + return score + + def decode_ctc(logits: np.ndarray, charset: Sequence[str]) -> Tuple[str, float]: idxs = logits.argmax(axis=1) confs = logits.max(axis=1) @@ -292,6 +344,7 @@ def recognize_crop( input_handle: Any, output_handle: Any, charset: Sequence[str], + variant_count: int = REC_VARIANT_COUNT, ) -> Tuple[str, float]: x, y, w, h = box h_img, w_img = image_bgr.shape[:2] @@ -300,12 +353,24 @@ def recognize_crop( x2 = min(w_img, x + w + CROP_PAD_X_RIGHT) y2 = min(h_img, y + h + CROP_PAD_Y) crop = image_bgr[y1:y2, x1:x2] - arr = preprocess_rec_input(crop) - input_handle.reshape(arr.shape) - input_handle.copy_from_cpu(arr) - predictor.run() - out = output_handle.copy_to_cpu()[0] - return decode_ctc(out, charset) + best_text = "" + best_conf = 0.0 + best_score = float("-inf") + + for variant in build_rec_crop_variants(crop, variant_count): + arr = preprocess_rec_input(variant) + input_handle.reshape(arr.shape) + input_handle.copy_from_cpu(arr) + predictor.run() + out = output_handle.copy_to_cpu()[0] + text, conf = decode_ctc(out, charset) + score = score_rec_text(text, conf) + if score > best_score: + best_text = text + best_conf = conf + best_score = score + + return best_text, best_conf def extract_alpha_token(text: str) -> str: @@ -565,6 +630,7 @@ def recognize_mnemonic_grid( output_handle: Any, charset: Sequence[str], num_rows: int = 6, + variant_count: int = REC_VARIANT_COUNT, ) -> Optional[Tuple[str, float]]: """Recognize a 2-column mnemonic grid with `num_rows` rows (num_rows*2 words total). @@ -592,10 +658,10 @@ def recognize_mnemonic_grid( return None left_text, left_conf = recognize_crop( - image_bgr, left_box, predictor, input_handle, output_handle, charset + image_bgr, left_box, predictor, input_handle, output_handle, charset, variant_count ) right_text, right_conf = recognize_crop( - image_bgr, right_box, predictor, input_handle, output_handle, charset + image_bgr, right_box, predictor, input_handle, output_handle, charset, variant_count ) left_token = extract_alpha_token(left_text) @@ -624,6 +690,7 @@ def recognize_generic_lines( input_handle: Any, output_handle: Any, charset: Sequence[str], + variant_count: int = REC_VARIANT_COUNT, ) -> Tuple[str, float]: lines: List[str] = [] confs: List[float] = [] @@ -632,7 +699,7 @@ def recognize_generic_lines( if box is None: continue text, conf = recognize_crop( - image_bgr, box, predictor, input_handle, output_handle, charset + image_bgr, box, predictor, input_handle, output_handle, charset, variant_count ) if text.strip(): lines.append(text.strip()) @@ -642,7 +709,7 @@ def recognize_generic_lines( # Last fallback: recognize the whole image as one line. h, w = image_bgr.shape[:2] text, conf = recognize_crop( - image_bgr, (0, 0, w, h), predictor, input_handle, output_handle, charset + image_bgr, (0, 0, w, h), predictor, input_handle, output_handle, charset, variant_count ) return text.strip(), conf * 100.0 @@ -670,6 +737,11 @@ def infer_once(payload: Dict[str, Any]) -> Dict[str, Any]: expected_word_count = 0 if expected_word_count not in {12, 18, 20, 24}: expected_word_count = 0 + try: + rec_variant_count = int(payload.get("recVariantCount") or REC_VARIANT_COUNT) + except (TypeError, ValueError): + rec_variant_count = REC_VARIANT_COUNT + rec_variant_count = max(1, min(4, rec_variant_count)) verify_max_index = expected_word_count if expected_word_count in {12, 18, 20, 24} else 12 if layout_hint in {"verify-number", "verify_number"}: @@ -695,7 +767,7 @@ def infer_once(payload: Dict[str, Any]) -> Dict[str, Any]: boxes = detect_text_boxes(image_bgr, mask_top_ratio=0.0, mask_bottom_ratio=1.0) rows = cluster_rows(boxes, image_bgr.shape[0]) text, confidence = recognize_generic_lines( - image_bgr, rows, predictor, input_handle, output_handle, charset + image_bgr, rows, predictor, input_handle, output_handle, charset, rec_variant_count ) detected_index = parse_word_index_from_text(text, verify_max_index) if detected_index != -1 and f"#{detected_index}" not in text: @@ -716,8 +788,9 @@ def infer_once(payload: Dict[str, Any]) -> Dict[str, Any]: predictor, input_handle, output_handle, charset = ensure_rec_model(resolve_en_model_dir()) mnemonic_layout = layout_hint in {"mnemonic", "mnemonic-grid", "mnemonic_words"} - # For mnemonic pages, keep full vertical content (18/24 need top and bottom indices). - use_full_vertical_mask = mnemonic_layout or expected_word_count >= 18 + verify_options_layout = layout_hint in {"verify-options", "verify_options"} + # Mnemonic and verify-option crops are already tight; keep full vertical content. + use_full_vertical_mask = mnemonic_layout or verify_options_layout or expected_word_count >= 18 boxes = detect_text_boxes( image_bgr, mask_top_ratio=0.0 if use_full_vertical_mask else 0.14, @@ -735,13 +808,17 @@ def infer_once(payload: Dict[str, Any]) -> Dict[str, Any]: # Known word count: try the exact grid layout. num_rows = MNEMONIC_GRID_WORD_COUNTS[expected_word_count] mnemonic_result = recognize_mnemonic_grid( - image_bgr, rows, predictor, input_handle, output_handle, charset, num_rows=num_rows + image_bgr, rows, predictor, input_handle, output_handle, charset, + num_rows=num_rows, + variant_count=rec_variant_count, ) else: # Unknown word count: try all supported layouts in ascending order. for num_rows in sorted(MNEMONIC_GRID_WORD_COUNTS.values()): mnemonic_result = recognize_mnemonic_grid( - image_bgr, rows, predictor, input_handle, output_handle, charset, num_rows=num_rows + image_bgr, rows, predictor, input_handle, output_handle, charset, + num_rows=num_rows, + variant_count=rec_variant_count, ) if mnemonic_result is not None: break @@ -763,6 +840,7 @@ def infer_once(payload: Dict[str, Any]) -> Dict[str, Any]: retry_result = recognize_mnemonic_grid( image_bgr, retry_rows, predictor, input_handle, output_handle, charset, num_rows=retry_num_rows, + variant_count=rec_variant_count, ) if retry_result is not None: boxes = retry_boxes @@ -777,7 +855,7 @@ def infer_once(payload: Dict[str, Any]) -> Dict[str, Any]: mode = "mnemonic-grid" else: text, confidence = recognize_generic_lines( - image_bgr, rows, predictor, input_handle, output_handle, charset + image_bgr, rows, predictor, input_handle, output_handle, charset, rec_variant_count ) return { @@ -790,6 +868,7 @@ def infer_once(payload: Dict[str, Any]) -> Dict[str, Any]: "mode": mode, "boxCount": len(boxes), "rowCount": len(rows), + "recVariantCount": rec_variant_count, } diff --git a/src/components/CameraPanel.tsx b/src/components/CameraPanel.tsx index be53755..04dab0f 100644 --- a/src/components/CameraPanel.tsx +++ b/src/components/CameraPanel.tsx @@ -3,6 +3,8 @@ import type { Worker } from 'tesseract.js'; import { validateMnemonic } from '@scure/bip39'; import { wordlist as bip39English } from '@scure/bip39/wordlists/english.js'; import { slip39English } from '../slip39Wordlist'; +import { DEVICE_OCR_SCENES, type SequenceOcrSceneConfig } from '../ocr/deviceScenes'; +import { getStoredDeviceTestSet } from '../deviceTestSetPreference'; import { rotateVideoFrameToCanvas, cropToROI, @@ -136,12 +138,58 @@ interface OcrTriggerOptions { mergeWithStored?: boolean; allowPartial?: boolean; requireBip39?: boolean; + sceneConfig?: SequenceOcrSceneConfig; +} + +interface VerifyOcrTriggerOptions { + numberSceneConfig?: SequenceOcrSceneConfig; + optionsSceneConfig?: SequenceOcrSceneConfig; } type VideoWithFrameCallback = HTMLVideoElement & { requestVideoFrameCallback?: (callback: () => void) => number; }; +function withOcrSceneDefaults( + sceneConfig: SequenceOcrSceneConfig | undefined, + fallback: OcrSceneConfig +): OcrSceneConfig | undefined { + if (!sceneConfig) return undefined; + return { + roi: sceneConfig.roi, + scale: sceneConfig.scale ?? fallback.scale, + useNearestNeighbor: sceneConfig.useNearestNeighbor ?? fallback.useNearestNeighbor, + }; +} + +function getCurrentDeviceOcrScenes() { + return DEVICE_OCR_SCENES[getStoredDeviceTestSet()]; +} + +function getCurrentDeviceRecVariantCount(layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic'): number | undefined { + return getStoredDeviceTestSet() === 'pro2' && (layoutHint === 'mnemonic' || layoutHint === 'verify-options') + ? 4 + : undefined; +} + +function getCurrentMnemonicSceneConfig(): OcrSceneConfig | undefined { + return withOcrSceneDefaults( + getCurrentDeviceOcrScenes().createWallet?.mnemonic12, + MNEMONIC_SCENE + ); +} + +function getCurrentVerifySceneConfigs(): { + numberSceneConfig?: OcrSceneConfig; + optionsSceneConfig?: OcrSceneConfig; +} { + const verifyWalletScenes = getCurrentDeviceOcrScenes().verifyWallet; + return { + numberSceneConfig: withOcrSceneDefaults(verifyWalletScenes?.number, VERIFY_NUMBER_SCENE), + optionsSceneConfig: withOcrSceneDefaults(verifyWalletScenes?.options, VERIFY_OPTIONS_SCENE), + }; +} + function waitForNextVideoFrame(video: HTMLVideoElement): Promise { return new Promise((resolve) => { const candidate = video as VideoWithFrameCallback; @@ -980,7 +1028,8 @@ function CameraPanel() { const paddle = await window.electronAPI.paddleOcrEnRecognize( ocrInputDataUrl, inferredLayoutHint, - options?.expectedWordCount + options?.expectedWordCount, + getCurrentDeviceRecVariantCount(inferredLayoutHint) ); const fallbackConfidence = Number.isFinite(paddle.confidence) ? paddle.confidence : 0; return { @@ -1041,49 +1090,51 @@ function CameraPanel() { onImageForOcr?: (dataUrl: string) => void; frameCanvas?: HTMLCanvasElement; maxIndex?: number; + sceneConfig?: OcrSceneConfig; }): Promise<{ number: number; rawText: string; confidence: number; imageDataUrl: string; preOcrImageDataUrl: string } | null> => { const maxIndex = Math.max(1, Math.min(24, Math.floor(options?.maxIndex ?? 12))); + const baseScene = options?.sceneConfig ?? VERIFY_NUMBER_SCENE; const fallbackScenes: OcrSceneConfig[] = [ // Tight first-line crop (often improves "#N" visibility). { - ...VERIFY_NUMBER_SCENE, + ...baseScene, roi: { - x: VERIFY_NUMBER_SCENE.roi.x, - y: Math.max(0, VERIFY_NUMBER_SCENE.roi.y - 20), - width: VERIFY_NUMBER_SCENE.roi.width, - height: Math.max(60, Math.round(VERIFY_NUMBER_SCENE.roi.height * 0.72)), + x: baseScene.roi.x, + y: Math.max(0, baseScene.roi.y - 20), + width: baseScene.roi.width, + height: Math.max(60, Math.round(baseScene.roi.height * 0.72)), }, scale: 7, }, // Extra-tight hash-right area to catch a faint single digit (e.g. "#1"). { - ...VERIFY_NUMBER_SCENE, + ...baseScene, roi: { - x: VERIFY_NUMBER_SCENE.roi.x + Math.round(VERIFY_NUMBER_SCENE.roi.width * 0.32), - y: Math.max(0, VERIFY_NUMBER_SCENE.roi.y - 20), - width: Math.max(120, Math.round(VERIFY_NUMBER_SCENE.roi.width * 0.45)), - height: Math.max(56, Math.round(VERIFY_NUMBER_SCENE.roi.height * 0.70)), + x: baseScene.roi.x + Math.round(baseScene.roi.width * 0.32), + y: Math.max(0, baseScene.roi.y - 20), + width: Math.max(120, Math.round(baseScene.roi.width * 0.45)), + height: Math.max(56, Math.round(baseScene.roi.height * 0.70)), }, scale: 8, }, - VERIFY_NUMBER_SCENE, + baseScene, { - ...VERIFY_NUMBER_SCENE, + ...baseScene, roi: { - x: Math.max(0, VERIFY_NUMBER_SCENE.roi.x - 40), - y: Math.max(0, VERIFY_NUMBER_SCENE.roi.y - 70), - width: VERIFY_NUMBER_SCENE.roi.width + 120, - height: VERIFY_NUMBER_SCENE.roi.height + 140, + x: Math.max(0, baseScene.roi.x - 40), + y: Math.max(0, baseScene.roi.y - 70), + width: baseScene.roi.width + 120, + height: baseScene.roi.height + 140, }, scale: 4, }, { - ...VERIFY_NUMBER_SCENE, + ...baseScene, roi: { - x: Math.max(0, VERIFY_NUMBER_SCENE.roi.x - 80), - y: Math.max(0, VERIFY_NUMBER_SCENE.roi.y - 120), - width: VERIFY_NUMBER_SCENE.roi.width + 220, - height: VERIFY_NUMBER_SCENE.roi.height + 260, + x: Math.max(0, baseScene.roi.x - 80), + y: Math.max(0, baseScene.roi.y - 120), + width: baseScene.roi.width + 220, + height: baseScene.roi.height + 260, }, scale: 3, }, @@ -1208,35 +1259,42 @@ function CameraPanel() { if (bestAttempt) { // Paddle pass failed to get a valid index; fallback to Tesseract digits-only across ROI variants. const tesseractCandidates: NumberCandidate[] = []; - for (const scene of fallbackScenes) { - const tesseractResult = await runOcrOnRegion(scene, { - onImageForOcr: options?.onImageForOcr, - frameCanvas: options?.frameCanvas, - forceTesseract: true, - ocrParams: NUMBER_OCR_PARAMS, - layoutHint: 'verify-number', - expectedWordCount: maxIndex, - }); - if (!tesseractResult) continue; + try { + for (const scene of fallbackScenes) { + const tesseractResult = await runOcrOnRegion(scene, { + onImageForOcr: options?.onImageForOcr, + frameCanvas: options?.frameCanvas, + forceTesseract: true, + ocrParams: NUMBER_OCR_PARAMS, + layoutHint: 'verify-number', + expectedWordCount: maxIndex, + }); + if (!tesseractResult) continue; - const tesseractIndex = extractWordIndexFromText(tesseractResult.rawText, maxIndex); - console.log( - `[CameraPanel] Number OCR tesseract fallback raw: "${tesseractResult.rawText.trim()}", confidence: ${tesseractResult.confidence.toFixed(0)}%, roi=(${scene.roi.x},${scene.roi.y},${scene.roi.width},${scene.roi.height})` - ); - if (tesseractIndex !== -1) { - tesseractCandidates.push({ - number: tesseractIndex, - rawText: tesseractResult.rawText, - confidence: tesseractResult.confidence, - imageDataUrl: tesseractResult.imageDataUrl, - preOcrImageDataUrl: tesseractResult.preOcrImageDataUrl, - score: scoreNumberCandidate( + const tesseractIndex = extractWordIndexFromText(tesseractResult.rawText, maxIndex); + console.log( + `[CameraPanel] Number OCR tesseract fallback raw: "${tesseractResult.rawText.trim()}", confidence: ${tesseractResult.confidence.toFixed(0)}%, roi=(${scene.roi.x},${scene.roi.y},${scene.roi.width},${scene.roi.height})` + ); + if (tesseractIndex !== -1) { + tesseractCandidates.push({ + number: tesseractIndex, + rawText: tesseractResult.rawText, + confidence: tesseractResult.confidence, + imageDataUrl: tesseractResult.imageDataUrl, + preOcrImageDataUrl: tesseractResult.preOcrImageDataUrl, + score: scoreNumberCandidate( tesseractResult.rawText, tesseractIndex, tesseractResult.confidence - ), - }); + ), + }); + } } + } catch (error) { + console.warn( + '[CameraPanel] Number OCR tesseract fallback failed:', + error instanceof Error ? error.message : error + ); } const bestTesseract = pickBestCandidate(tesseractCandidates); @@ -1266,6 +1324,7 @@ function CameraPanel() { onImageForOcr?: (dataUrl: string) => void; frameCanvas?: HTMLCanvasElement; targetWord?: string; + sceneConfig?: OcrSceneConfig; }): Promise<{ rawText: string; rawOptions: string[]; @@ -1273,25 +1332,26 @@ function CameraPanel() { imageDataUrl: string; preOcrImageDataUrl: string; } | null> => { + const baseScene = options?.sceneConfig ?? VERIFY_OPTIONS_SCENE; const fallbackScenes: OcrSceneConfig[] = [ - VERIFY_OPTIONS_SCENE, + baseScene, { - ...VERIFY_OPTIONS_SCENE, + ...baseScene, roi: { - x: Math.max(0, VERIFY_OPTIONS_SCENE.roi.x - 40), - y: Math.max(0, VERIFY_OPTIONS_SCENE.roi.y - 40), - width: VERIFY_OPTIONS_SCENE.roi.width + 80, - height: VERIFY_OPTIONS_SCENE.roi.height + 120, + x: Math.max(0, baseScene.roi.x - 40), + y: Math.max(0, baseScene.roi.y - 40), + width: baseScene.roi.width + 80, + height: baseScene.roi.height + 120, }, scale: 4, }, { - ...VERIFY_OPTIONS_SCENE, + ...baseScene, roi: { - x: Math.max(0, VERIFY_OPTIONS_SCENE.roi.x - 80), - y: Math.max(0, VERIFY_OPTIONS_SCENE.roi.y - 90), - width: VERIFY_OPTIONS_SCENE.roi.width + 160, - height: VERIFY_OPTIONS_SCENE.roi.height + 220, + x: Math.max(0, baseScene.roi.x - 80), + y: Math.max(0, baseScene.roi.y - 90), + width: baseScene.roi.width + 160, + height: baseScene.roi.height + 220, }, scale: 3, }, @@ -1821,9 +1881,10 @@ function CameraPanel() { try { await waitForVideoToSettle(video); + const mnemonicSceneConfig = getCurrentMnemonicSceneConfig(); if (RAW_MNEMONIC_OCR_DEBUG_ONLY) { - const rawResult = await runOcrOnRegion(MNEMONIC_SCENE, { + const rawResult = await runOcrOnRegion(mnemonicSceneConfig ?? MNEMONIC_SCENE, { ocrParams: MNEMONIC_OCR_PARAMS, }); if (!rawResult) { @@ -1854,7 +1915,7 @@ function CameraPanel() { for (let attempt = 1; attempt <= MAX_OCR_RETRIES; attempt++) { console.log(`OCR attempt ${attempt}/${MAX_OCR_RETRIES}...`); - const result = await runSingleOcr(); + const result = await runSingleOcr({ sceneConfig: mnemonicSceneConfig }); if (!result) { console.warn('OCR attempt failed'); @@ -2306,6 +2367,7 @@ function CameraPanel() { const mergeWithStored = !!triggerOptions.mergeWithStored; const allowPartial = !!triggerOptions.allowPartial; const requireBip39 = triggerOptions.requireBip39 ?? true; + const triggerSceneConfig = withOcrSceneDefaults(triggerOptions.sceneConfig, MNEMONIC_SCENE); console.log( `[CameraPanel] External OCR trigger received (expected=${expectedWordCount}, merge=${mergeWithStored}, allowPartial=${allowPartial}, requireBip39=${requireBip39})` @@ -2381,6 +2443,7 @@ function CameraPanel() { ); } }, + sceneConfig: triggerSceneConfig, expectedWordCount, applyBip39Wordlist: requireBip39, }); @@ -2606,7 +2669,11 @@ function CameraPanel() { // Listen for verification OCR trigger events (from ControlPanel verification steps) useEffect(() => { - const handleTriggerVerifyOcr = async () => { + const handleTriggerVerifyOcr = async (event: Event) => { + const triggerOptions = + ((event as CustomEvent).detail ?? {}) as VerifyOcrTriggerOptions; + const numberSceneConfig = withOcrSceneDefaults(triggerOptions.numberSceneConfig, VERIFY_NUMBER_SCENE); + const optionsSceneConfig = withOcrSceneDefaults(triggerOptions.optionsSceneConfig, VERIFY_OPTIONS_SCENE); console.log('[CameraPanel] Verification OCR trigger received'); setIsOcrProcessing(true); setFullFrameImageUrl(null); @@ -2665,6 +2732,7 @@ function CameraPanel() { const numberResult = await runNumberOcr({ frameCanvas, maxIndex: verifyMaxIndex, + sceneConfig: numberSceneConfig, onImageForOcr: (dataUrl) => { if (!verifyNumberSaved && window.electronAPI?.saveCaptureToDownloads) { verifyNumberSaved = true; @@ -2693,6 +2761,7 @@ function CameraPanel() { const optionsResult = await runVerifyOptionsOcr({ frameCanvas, targetWord, + sceneConfig: optionsSceneConfig, onImageForOcr: (dataUrl) => { if (!verifyOptionsSaved && window.electronAPI?.saveCaptureToDownloads) { verifyOptionsSaved = true; @@ -2929,7 +2998,11 @@ function CameraPanel() { // Manually trigger verification-page OCR debug flow (single-frame dual ROI). const handleVerifyDebugOcr = () => { - window.dispatchEvent(new CustomEvent('qa-auto-hw:trigger-verify-ocr')); + window.dispatchEvent( + new CustomEvent('qa-auto-hw:trigger-verify-ocr', { + detail: getCurrentVerifySceneConfigs(), + }) + ); }; return ( diff --git a/src/components/ControlPanel.css b/src/components/ControlPanel.css index 1d0019d..be1c85a 100644 --- a/src/components/ControlPanel.css +++ b/src/components/ControlPanel.css @@ -126,6 +126,53 @@ margin: 0; } +.sequence-header-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--spacing-sm); +} + +.sequence-title-group { + display: inline-flex; + align-items: center; + gap: var(--spacing-xs); +} + +.sequence-device-badge, +.seq-btn-device { + display: inline-flex; + align-items: center; + height: 18px; + padding: 0 6px; + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + color: var(--color-text-muted); + background: var(--color-surface); + font-size: 0.6875rem; + line-height: 1; + flex-shrink: 0; +} + +.device-test-set-select { + display: inline-flex; + align-items: center; + gap: var(--spacing-xs); + font-size: 0.75rem; + color: var(--color-text-muted); +} + +.device-test-set-select select { + height: 28px; + padding: 0 var(--spacing-sm); + background: var(--color-background); + border: 1px solid var(--color-border); + border-radius: var(--radius-md); + color: var(--color-text); + font-size: 0.8125rem; + -webkit-app-region: no-drag; +} + .control-selectors { display: flex; justify-content: center; diff --git a/src/components/ControlPanel.tsx b/src/components/ControlPanel.tsx index 31396fc..7ade623 100644 --- a/src/components/ControlPanel.tsx +++ b/src/components/ControlPanel.tsx @@ -13,14 +13,21 @@ import './ControlPanel.css'; * Both QA Auto Hardware UI and MCP tools use the same sequence definitions. */ import { - getAllSequenceIds, - getSequence, - getFullSteps, - getAllCategories, - getSequencesByCategory, type AutoSequence, type AutoStep, } from '../../electron/mcp/sequences'; +import { + DEFAULT_DEVICE_TEST_SET_ID, + DEVICE_TEST_SETS, + getAllCategories, + getDeviceHomeCoord, + getAllSequenceIds, + getFullSteps, + getSequence, + getSequencesByCategory, + type DeviceTestSetId, +} from '../../electron/mcp/sequenceSets'; +import { DEVICE_OCR_SCENES } from '../ocr/deviceScenes'; import { executeClickStep, executeSwipeStep } from '../../electron/mcp/utils/executeStep'; import { getAutomationPresetEntries, @@ -28,14 +35,15 @@ import { type AutomationPresetSuite, } from '../../electron/mcp/automationActionPresets'; import { executeDeviceActionSequence } from '../../electron/mcp/deviceActionRuntime'; +import { getStoredDeviceTestSet, storeDeviceTestSet } from '../deviceTestSetPreference'; // Get all sequences from the shared definition -const OPERATION_SEQUENCES: AutoSequence[] = getAllSequenceIds() - .map((id: string) => getSequence(id)) +const DEFAULT_OPERATION_SEQUENCES: AutoSequence[] = getAllSequenceIds(DEFAULT_DEVICE_TEST_SET_ID) + .map((id: string) => getSequence(id, DEFAULT_DEVICE_TEST_SET_ID)) .filter((seq): seq is AutoSequence => seq !== undefined); // Get all categories for the sequence panel -const SEQUENCE_CATEGORIES = getAllCategories(); +const DEFAULT_SEQUENCE_CATEGORIES = getAllCategories(DEFAULT_DEVICE_TEST_SET_ID); const SECURITY_CHECK_PRESETS = getAutomationPresetEntries('securityCheck'); const CHAIN_METHOD_BATCH_PRESETS = getAutomationPresetEntries('chainMethodBatch'); @@ -57,6 +65,7 @@ interface ControlPanelState { activeOperationKey: string; selectedSequenceId: string; selectedCategory: string; + selectedDeviceTestSet: DeviceTestSetId; /** Words captured via OCR during create-wallet flow */ capturedWords: string[]; } @@ -100,6 +109,7 @@ interface SequenceExecutionResult { message: string; sequenceId: string; sequenceName?: string; + deviceTestSetId?: string; stepsCompleted: number; totalSteps: number; mnemonicState?: { @@ -119,25 +129,33 @@ function hasSwipeTarget(step: AutoStep): step is AutoStep & { swipeTo: { x: numb } function ControlPanel() { - const [state, setState] = useState({ - isConnected: false, - resourceHandle: 0, - serverIP: ARM_CONTROLLER_CONFIG.defaultServerIP, - comPort: ARM_CONTROLLER_CONFIG.defaultComPort, - stepSize: ARM_CONTROLLER_CONFIG.defaultStepSize, - zDepth: ARM_CONTROLLER_CONFIG.defaultZDepth, - currentX: 0, - currentY: 0, - isLoading: false, - isReady: false, - error: null, - isAutoRunning: false, - autoProgress: 0, - autoTotalSteps: 0, - activeOperationKey: '', - selectedSequenceId: OPERATION_SEQUENCES[0].id, - selectedCategory: SEQUENCE_CATEGORIES[0], - capturedWords: [], + const [state, setState] = useState(() => { + const selectedDeviceTestSet = getStoredDeviceTestSet(); + const sequenceCategories = getAllCategories(selectedDeviceTestSet); + const selectedCategory = sequenceCategories[0] ?? DEFAULT_SEQUENCE_CATEGORIES[0] ?? ''; + const selectedSequence = getSequencesByCategory(selectedCategory, selectedDeviceTestSet)[0]; + + return { + isConnected: false, + resourceHandle: 0, + serverIP: ARM_CONTROLLER_CONFIG.defaultServerIP, + comPort: ARM_CONTROLLER_CONFIG.defaultComPort, + stepSize: ARM_CONTROLLER_CONFIG.defaultStepSize, + zDepth: ARM_CONTROLLER_CONFIG.defaultZDepth, + currentX: 0, + currentY: 0, + isLoading: false, + isReady: false, + error: null, + isAutoRunning: false, + autoProgress: 0, + autoTotalSteps: 0, + activeOperationKey: '', + selectedSequenceId: selectedSequence?.id ?? DEFAULT_OPERATION_SEQUENCES[0].id, + selectedCategory, + selectedDeviceTestSet, + capturedWords: [], + }; }); // Ref to track if auto operation should be cancelled @@ -145,6 +163,10 @@ function ControlPanel() { const [logs, setLogs] = useState([]); + useEffect(() => { + storeDeviceTestSet(state.selectedDeviceTestSet); + }, [state.selectedDeviceTestSet]); + const addLog = useCallback((action: string, detail: string) => { const now = new Date(); const time = now.toLocaleTimeString('zh-CN', { hour12: false }); @@ -154,11 +176,14 @@ function ControlPanel() { ]); }, []); - const resolveSequenceStepsForUi = useCallback(async (sequence: AutoSequence): Promise => { + const resolveSequenceStepsForUi = useCallback(async ( + sequence: AutoSequence, + deviceTestSetId: DeviceTestSetId + ): Promise => { if (window.electronAPI?.resolveSequenceSteps) { - return window.electronAPI.resolveSequenceSteps(sequence.id); + return window.electronAPI.resolveSequenceSteps(sequence.id, deviceTestSetId); } - return getFullSteps(sequence); + return getFullSteps(sequence, deviceTestSetId); }, []); /** @@ -540,6 +565,46 @@ function ControlPanel() { } }; + const handleResetPosition = async () => { + if (state.isLoading || !state.isConnected || !state.isReady || state.isAutoRunning) return; + + const homeCoord = getDeviceHomeCoord(state.selectedDeviceTestSet); + setState(prev => ({ ...prev, isLoading: true, error: null })); + + try { + await sendCommand({ + duankou: '0', + hco: state.resourceHandle, + daima: `Z${ARM_CONTROLLER_CONFIG.zUp}`, + }); + await sendCommand({ + duankou: '0', + hco: state.resourceHandle, + daima: `X${homeCoord.x}Y${homeCoord.y}`, + }); + + addLog('复位', `${state.selectedDeviceTestSet} home (${homeCoord.x},${homeCoord.y})`); + setState(prev => ({ + ...prev, + currentX: homeCoord.x, + currentY: homeCoord.y, + isLoading: false, + })); + await syncArmStateToMain({ + currentX: homeCoord.x, + currentY: homeCoord.y, + zDepth: ARM_CONTROLLER_CONFIG.zUp, + }); + } catch (error) { + addLog('错误', `复位失败: ${error instanceof Error ? error.message : 'Unknown'}`); + setState(prev => ({ + ...prev, + isLoading: false, + error: error instanceof Error ? error.message : 'Reset position failed', + })); + } + }; + /** * Executes an auto operation sequence. * If sequenceId is provided, runs that sequence; otherwise uses the currently selected one. @@ -548,14 +613,16 @@ function ControlPanel() { async ( targetId: string, armContext: SequenceExecutionArmContext, + deviceTestSetId: DeviceTestSetId, options?: { updateSelectedSequence?: boolean } ): Promise => { - const sequence = OPERATION_SEQUENCES.find((s) => s.id === targetId); + const sequence = getSequence(targetId, deviceTestSetId); if (!sequence) { return { success: false, - message: `Unknown sequence ID: ${targetId}`, + message: `Unknown ${deviceTestSetId} sequence ID: ${targetId}`, sequenceId: targetId, + deviceTestSetId, stepsCompleted: 0, totalSteps: 0, }; @@ -565,7 +632,7 @@ function ControlPanel() { setState((prev) => ({ ...prev, selectedSequenceId: targetId })); } - const steps = await resolveSequenceStepsForUi(sequence); + const steps = await resolveSequenceStepsForUi(sequence, deviceTestSetId); const totalVerifySteps = steps.filter((step) => !!step.ocrVerify).length; let finishedVerifySteps = 0; let stepsCompleted = 0; @@ -583,7 +650,7 @@ function ControlPanel() { error: null, capturedWords: [], })); - addLog('自动', `开始执行自动操作序列: ${sequence.name}`); + addLog('自动', `开始执行 ${deviceTestSetId} 自动操作序列: ${sequence.name}`); const send = async (daima: string) => { await sendCommandToServer(armContext.serverIP, { @@ -603,6 +670,7 @@ function ControlPanel() { message: `Sequence "${sequence.name}" stopped by user at step ${stepsCompleted + 1}`, sequenceId: targetId, sequenceName: sequence.name, + deviceTestSetId, stepsCompleted, totalSteps: steps.length, }; @@ -611,7 +679,10 @@ function ControlPanel() { const step = steps[i]; setState((prev) => ({ ...prev, autoProgress: i + 1 })); - if (step.ocrVerify) { + if (step.moveOnly) { + await send(`X${step.x}Y${step.y}`); + addLog('自动', `${step.label} - 移动到 (${step.x},${step.y})`); + } else if (step.ocrVerify) { const verifyRound = finishedVerifySteps + 1; addLog('验证', `开始第 ${verifyRound}/${totalVerifySteps} 次确认题 OCR`); await send(`X${step.x}Y${step.y}`); @@ -625,7 +696,15 @@ function ControlPanel() { resolve((e as CustomEvent).detail); }; window.addEventListener('qa-auto-hw:verify-ocr-result', handler); - window.dispatchEvent(new CustomEvent('qa-auto-hw:trigger-verify-ocr')); + const verifyScenes = DEVICE_OCR_SCENES[deviceTestSetId].verifyWallet; + window.dispatchEvent( + new CustomEvent('qa-auto-hw:trigger-verify-ocr', { + detail: { + numberSceneConfig: verifyScenes?.number, + optionsSceneConfig: verifyScenes?.options, + }, + }) + ); }), new Promise((resolve) => setTimeout( @@ -786,6 +865,7 @@ function ControlPanel() { message: `Sequence "${sequence.name}" completed successfully`, sequenceId: targetId, sequenceName: sequence.name, + deviceTestSetId, stepsCompleted, totalSteps: steps.length, mnemonicState, @@ -802,6 +882,7 @@ function ControlPanel() { message: `Sequence execution failed at step ${stepsCompleted + 1}: ${message}`, sequenceId: targetId, sequenceName: sequence.name, + deviceTestSetId, stepsCompleted, totalSteps: steps.length, }; @@ -822,11 +903,16 @@ function ControlPanel() { if (state.isLoading || !state.isConnected || !state.isReady || state.isAutoRunning) return; const targetId = sequenceId || state.selectedSequenceId; - await runSequenceExecution(targetId, { - isConnected: state.isConnected, - resourceHandle: state.resourceHandle, - serverIP: state.serverIP, - }, { updateSelectedSequence: !!sequenceId }); + await runSequenceExecution( + targetId, + { + isConnected: state.isConnected, + resourceHandle: state.resourceHandle, + serverIP: state.serverIP, + }, + state.selectedDeviceTestSet, + { updateSelectedSequence: !!sequenceId } + ); }, [ runSequenceExecution, state.isAutoRunning, @@ -835,6 +921,7 @@ function ControlPanel() { state.isReady, state.resourceHandle, state.selectedSequenceId, + state.selectedDeviceTestSet, state.serverIP, ]); @@ -844,18 +931,24 @@ function ControlPanel() { } return window.electronAPI.onMcpExecuteSequenceRequest(async (payload) => { + const requestedDeviceTestSet = payload.deviceTestSetId === 'pro2' ? 'pro2' : DEFAULT_DEVICE_TEST_SET_ID; if (state.isAutoRunning) { window.electronAPI?.sendMcpExecuteSequenceResponse?.({ success: false, message: 'Renderer sequence execution is already running', sequenceId: payload.sequenceId, + deviceTestSetId: requestedDeviceTestSet, stepsCompleted: 0, totalSteps: 0, }); return; } - const result = await runSequenceExecution(payload.sequenceId, payload.armState); + const result = await runSequenceExecution( + payload.sequenceId, + payload.armState, + requestedDeviceTestSet + ); window.electronAPI?.sendMcpExecuteSequenceResponse?.(result); }); }, [runSequenceExecution, state.isAutoRunning]); @@ -933,7 +1026,14 @@ function ControlPanel() { const isControlDisabled = !state.isConnected || !state.isReady || state.isLoading || state.isAutoRunning; - const categorySequences = getSequencesByCategory(state.selectedCategory); + const sequenceCategories = getAllCategories(state.selectedDeviceTestSet); + const activeSequenceCategory = sequenceCategories.includes(state.selectedCategory) + ? state.selectedCategory + : sequenceCategories[0] ?? ''; + const categorySequences = getSequencesByCategory(activeSequenceCategory, state.selectedDeviceTestSet); + const selectedDeviceTestSetName = DEVICE_TEST_SETS.find( + device => device.id === state.selectedDeviceTestSet + )?.name ?? state.selectedDeviceTestSet; const capturedFilledCount = state.capturedWords.filter((word) => !!word).length; return ( @@ -962,6 +1062,14 @@ function ControlPanel() { X: {state.currentX} Y: {state.currentY} + ))} -
+
{categorySequences.map(seq => { const isRunning = state.isAutoRunning && state.activeOperationKey === `sequence:${seq.id}`; return (