Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -74,4 +74,6 @@ scripts/.venv/
# Mnemonic secrets (use mnemonics.example.json as template)
mnemonics.local.json

./claude/worktrees/*
./claude/worktrees/*
# Python bytecode
scripts/__pycache__/
22 changes: 14 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,8 @@ yarn setup:ocr
# Download OCR models
python3 - <<'PY'
from huggingface_hub import snapshot_download
snapshot_download('PaddlePaddle/en_PP-OCRv5_mobile_rec', local_dir='models/paddle_ocr/en_PP-OCRv5_mobile_rec')
snapshot_download('PaddlePaddle/PP-OCRv5_mobile_det', local_dir='models/paddle_ocr/PP-OCRv5_mobile_det')
snapshot_download('PaddlePaddle/PP-OCRv5_mobile_rec', local_dir='models/paddle_ocr/PP-OCRv5_mobile_rec')
snapshot_download('PaddlePaddle/PP-OCRv6_medium_rec_onnx', local_dir='models/paddle_ocr/PP-OCRv6_medium_rec_onnx')
snapshot_download('PaddlePaddle/PP-OCRv6_medium_det_onnx', local_dir='models/paddle_ocr/PP-OCRv6_medium_det_onnx')
print('done')
PY

Expand All @@ -131,8 +130,8 @@ yarn build:linux # Linux

QA Auto Hardware uses a dual-path OCR pipeline for hardware wallet mnemonic capture:

- **Mnemonic display page & confirmation option area**: `en_PP-OCRv5_mobile_rec`
- **Confirmation page question number area (`#N`)**: `PP-OCRv5_mobile_det` + `PP-OCRv5_mobile_rec`
- **Mnemonic display page & confirmation option area**: `PP-OCRv6_medium_rec_onnx`
- **Confirmation page question number area (`#N`)**: `PP-OCRv6_medium_det_onnx` + `PP-OCRv6_medium_rec_onnx`

**Typical workflow:**

Expand All @@ -145,9 +144,16 @@ QA Auto Hardware uses a dual-path OCR pipeline for hardware wallet mnemonic capt

| Model | Environment Variable Override |
|-------|-------------------------------|
| `en_PP-OCRv5_mobile_rec` | `QA_AUTO_HW_OCR_MODEL_DIR` |
| `PP-OCRv5_mobile_det` | `QA_AUTO_HW_OCR_DET_MODEL_DIR` |
| `PP-OCRv5_mobile_rec` | `QA_AUTO_HW_OCR_MULTI_REC_MODEL_DIR` |
| `PP-OCRv6_medium_rec_onnx` | `QA_AUTO_HW_OCR_MODEL_DIR` |
| `PP-OCRv6_medium_det_onnx` | `QA_AUTO_HW_OCR_DET_MODEL_DIR` |

To try the smaller PP-OCRv6 small models:

```bash
QA_AUTO_HW_OCR_MODEL_DIR=models/paddle_ocr/PP-OCRv6_small_rec_onnx \
QA_AUTO_HW_OCR_DET_MODEL_DIR=models/paddle_ocr/PP-OCRv6_small_det_onnx \
yarn electron:dev
```

Optional tuning: `QA_AUTO_HW_OCR_MAX_IMAGE_SIDE` (default 1280), `QA_AUTO_HW_OCR_CPU_THREADS` (default 4)

Expand Down
3 changes: 2 additions & 1 deletion electron/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,7 @@ ipcMain.handle(
);

/**
* IPC handler: Runs en_PP-OCRv5_mobile_rec inference in a Python subprocess.
* IPC handler: Runs PP-OCRv6 small OCR inference in a Python subprocess.
*/
ipcMain.handle(
'paddleocr-en-recognize',
Expand All @@ -677,6 +677,7 @@ ipcMain.handle(
layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic';
expectedWordCount?: number;
recVariantCount?: number;
wordlistHint?: 'bip39' | 'slip39';
}
) => runPaddleOcrEn(payload)
);
Expand Down
5 changes: 3 additions & 2 deletions electron/mcp/sequences.ts
Original file line number Diff line number Diff line change
Expand Up @@ -316,12 +316,13 @@ const ALL_PAGE_ACTIONS: PageAction[] = [
name: '输入PIN码1111',
group: '初始设置',
steps: [
{ label: '输入PIN码1', x: 25, y: 50, depth: 12 },
// Wait for the PIN pad to finish rendering before the first tap.
{ label: '输入PIN码1', x: 25, y: 50, depth: 12, delayBefore: 500 },
{ label: '输入PIN码2', x: 25, y: 50, depth: 12 },
{ label: '输入PIN码3', x: 25, y: 50, depth: 12 },
{ label: '输入PIN码4', x: 25, y: 50, depth: 12 },
{ label: '点击确认', x: 55, y: 85, depth: 12 },
{ label: '再次确认PIN码1', x: 25, y: 50, depth: 12 },
{ label: '再次确认PIN码1', x: 25, y: 50, depth: 12, delayBefore: 500 },
{ label: '再次确认PIN码2', x: 25, y: 50, depth: 12 },
{ label: '再次确认PIN码3', x: 25, y: 50, depth: 12 },
{ label: '再次确认PIN码4', x: 25, y: 50, depth: 12 },
Expand Down
1 change: 1 addition & 0 deletions electron/mcp/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export interface MnemonicOcrRequest {
allowPartial?: boolean;
requireBip39?: boolean;
sceneConfig?: SequenceOcrSceneConfig;
deviceTestSetId?: 'pro' | 'pro2';
}

export interface MnemonicStoreMetadata {
Expand Down
9 changes: 7 additions & 2 deletions electron/mcp/tools/executeSequence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
getPageAction,
getSequence,
normalizeDeviceTestSetId,
type DeviceTestSetId,
} from '../sequenceSets';
import { resolvePageActionStepsForDevice } from '../sequenceSetResolver';
import { saveCaptureToDownloads } from '../../saveCapture';
Expand Down Expand Up @@ -90,6 +91,7 @@ function requiresPostSequenceCooldown(sequence: { category: string; actions: str
async function executeStep(
step: AutoStep,
resourceHandle: number,
deviceTestSetId: DeviceTestSetId,
httpRequest: (url: string) => Promise<string>
): Promise<void> {
if ((step.delayBefore ?? 0) > 0) {
Expand All @@ -114,7 +116,10 @@ async function executeStep(
// Give the device screen time to finish inertial scrolling before capture.
await delay(1600);

const ocrResult = await runMnemonicOcr(ocrCaptureConfig);
const ocrResult = await runMnemonicOcr({
...ocrCaptureConfig,
deviceTestSetId,
});
if (!ocrResult) {
throw new Error('Renderer did not return mnemonic OCR result');
}
Expand Down Expand Up @@ -327,7 +332,7 @@ export async function executeExecuteSequence(
};
}

await executeStep(step, state.resourceHandle, httpRequest);
await executeStep(step, state.resourceHandle, deviceTestSetId, httpRequest);
stepsCompleted++;
console.log(`[execute-sequence] Step ${stepsCompleted}/${totalSteps}: ${step.label}`);
}
Expand Down
45 changes: 29 additions & 16 deletions electron/paddleOcrEn.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ export interface PaddleOcrEnRequest {
layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic';
expectedWordCount?: number;
recVariantCount?: number;
/** Closed wordlist of the current flow (BIP39 vs SLIP39 — never mixed). */
wordlistHint?: 'bip39' | 'slip39';
}

export interface PaddleOcrEnResult {
text: string;
confidence: number;
backend: 'en_PP-OCRv5_mobile_rec';
backend: 'PP-OCRv6_medium_rec_onnx';
elapsedMs: number;
}

Expand All @@ -34,26 +36,32 @@ const HEALTH_CACHE_TTL_MS = 30 * 1000;
const REQUIRED_PYTHON_MODULES = [
'paddleocr',
'paddle',
'onnxruntime',
'cv2',
'PIL',
'yaml',
'huggingface_hub',
] as const;
const REQUIRED_MODEL_CONFIGS = [
{
name: 'en_PP-OCRv5_mobile_rec',
envKeys: ['QA_AUTO_HW_OCR_MODEL_DIR', 'QA_AUTO_HW_EN_OCR_MODEL_DIR'],
name: 'PP-OCRv6_medium_rec_onnx',
envKeys: [
'QA_AUTO_HW_OCR_MODEL_DIR',
'QA_AUTO_HW_EN_OCR_MODEL_DIR',
'QA_AUTO_HW_OCR_MULTI_REC_MODEL_DIR',
],
},
{
name: 'PP-OCRv5_mobile_rec',
envKeys: ['QA_AUTO_HW_OCR_MULTI_REC_MODEL_DIR'],
name: 'PP-OCRv6_medium_det_onnx',
envKeys: ['QA_AUTO_HW_OCR_DET_MODEL_DIR'],
},
] as const;
const REQUIRED_MODEL_FILE_CONFIGS = [
{
name: 'PP-OCRv5_mobile_det',
envKeys: ['QA_AUTO_HW_OCR_DET_MODEL_DIR'],
suffixes: ['inference.json', 'inference.onnx', 'inference.yml'],
},
] as const;
const REQUIRED_MODEL_FILES = ['inference.json', 'inference.pdiparams', 'inference.yml'] as const;
const REQUIRED_MODEL_FILES = REQUIRED_MODEL_FILE_CONFIGS[0].suffixes;

let cachedHealthStatus: { expiresAt: number; value: PaddleOcrEnHealthStatus } | null = null;

Expand Down Expand Up @@ -271,14 +279,14 @@ class PaddleOcrEnDaemon {
await this.ensureStarted();

if (!this.child) {
throw new Error('PP-OCRv5 rec daemon is not running');
throw new Error('PP-OCRv6 rec daemon is not running');
}

const id = this.nextId++;
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
this.pending.delete(id);
reject(new Error('PP-OCRv5 rec request timed out after 2 minutes'));
reject(new Error('PP-OCRv6 rec request timed out after 2 minutes'));
}, 2 * 60 * 1000);

this.pending.set(id, { resolve, reject, timer });
Expand All @@ -294,7 +302,7 @@ class PaddleOcrEnDaemon {
}
this.startPromise = null;
this.stdoutBuffer = '';
this.rejectAllPending(new Error('PP-OCRv5 rec daemon stopped'));
this.rejectAllPending(new Error('PP-OCRv6 rec daemon stopped'));
}

private async ensureStarted(): Promise<void> {
Expand Down Expand Up @@ -335,7 +343,7 @@ class PaddleOcrEnDaemon {

const bootTimeout = setTimeout(() => {
child.kill('SIGKILL');
reject(new Error('Timed out while starting PP-OCRv5 rec daemon'));
reject(new Error('Timed out while starting PP-OCRv6 rec daemon'));
}, 45 * 1000);

child.stdout.on('data', (chunk: Buffer) => {
Expand All @@ -359,14 +367,14 @@ class PaddleOcrEnDaemon {
child.on('error', (err) => {
clearTimeout(bootTimeout);
this.child = null;
reject(new Error(`Failed to start PP-OCRv5 rec daemon: ${err.message}`));
reject(new Error(`Failed to start PP-OCRv6 rec daemon: ${err.message}`));
});

child.on('close', (code) => {
clearTimeout(bootTimeout);
this.child = null;
this.startPromise = null;
const reason = `PP-OCRv5 rec daemon exited with code ${code}. stderr: ${this.stderrBuffer.trim() || '(empty)'}`;
const reason = `PP-OCRv6 rec daemon exited with code ${code}. stderr: ${this.stderrBuffer.trim() || '(empty)'}`;
this.rejectAllPending(new Error(reason));
});
}).finally(() => {
Expand All @@ -391,6 +399,11 @@ class PaddleOcrEnDaemon {
}

if (parsed.type === 'ready') {
console.log(
`[paddleOcrEn] OCR server ready (script version: ${
typeof parsed.scriptVersion === 'string' ? parsed.scriptVersion : 'pre-versioning'
})`
);
options?.onReady?.();
continue;
}
Expand All @@ -407,15 +420,15 @@ class PaddleOcrEnDaemon {
const errorText =
typeof parsed.error === 'string'
? parsed.error
: 'Unknown PP-OCRv5 rec daemon error';
: 'Unknown PP-OCRv6 rec daemon error';
pending.reject(new Error(errorText));
continue;
}

pending.resolve({
text: typeof parsed.text === 'string' ? parsed.text : '',
confidence: typeof parsed.confidence === 'number' ? parsed.confidence : 0,
backend: 'en_PP-OCRv5_mobile_rec',
backend: 'PP-OCRv6_medium_rec_onnx',
elapsedMs: typeof parsed.elapsedMs === 'number' ? parsed.elapsedMs : 0,
});
}
Expand Down
15 changes: 12 additions & 3 deletions electron/preload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ interface VerifyOcrPayload {
interface PaddleOcrEnPayload {
text: string;
confidence: number;
backend: 'en_PP-OCRv5_mobile_rec';
backend: 'PP-OCRv6_medium_rec_onnx';
elapsedMs: number;
}

Expand All @@ -88,6 +88,12 @@ interface ResolvedSequenceStepPayload {
mergeWithStored?: boolean;
allowPartial?: boolean;
requireBip39?: boolean;
sceneConfig?: {
roi: { x: number; y: number; width: number; height: number };
scale?: number;
useNearestNeighbor?: boolean;
};
deviceTestSetId?: 'pro' | 'pro2';
};
ocrVerify?: {
options: { x: number; y: number; depth: number }[];
Expand Down Expand Up @@ -237,13 +243,15 @@ contextBridge.exposeInMainWorld('electronAPI', {
imageDataUrl: string,
layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic',
expectedWordCount?: number,
recVariantCount?: number
recVariantCount?: number,
wordlistHint?: 'bip39' | 'slip39'
) =>
ipcRenderer.invoke('paddleocr-en-recognize', {
imageDataUrl,
layoutHint,
expectedWordCount,
recVariantCount,
wordlistHint,
}) as Promise<PaddleOcrEnPayload>,

// MCP Log: Receive log entries from main process
Expand Down Expand Up @@ -327,7 +335,8 @@ declare global {
imageDataUrl: string,
layoutHint?: 'mnemonic' | 'verify-options' | 'verify-number' | 'generic',
expectedWordCount?: number,
recVariantCount?: number
recVariantCount?: number,
wordlistHint?: 'bip39' | 'slip39'
) => Promise<PaddleOcrEnPayload>;
onMcpServerReady: (callback: (info: { port: number }) => void) => void;
// MCP Logs
Expand Down
15 changes: 12 additions & 3 deletions electron/saveCapture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,20 @@ function extractBase64(dataUrlOrBase64: string): string {
return dataUrlOrBase64;
}

/**
* Picks the file extension matching the data URL mime type (default jpg).
*/
function extensionForDataUrl(dataUrlOrBase64: string): string {
const match = dataUrlOrBase64.match(/^data:image\/(png|jpeg|webp)/);
if (!match) return 'jpg';
return match[1] === 'jpeg' ? 'jpg' : match[1];
}

/**
* Saves a capture (data URL or raw base64) to the user's Downloads folder.
* Filename: qa-auto-hw-YYYYMMDD-HHmmss-sss-{contentHint}.jpg
* Filename: qa-auto-hw-YYYYMMDD-HHmmss-sss-{contentHint}.{ext}
*
* @param dataUrlOrBase64 - JPEG as data URL or raw base64 string
* @param dataUrlOrBase64 - image as data URL or raw base64 string (assumed JPEG)
* @param contentHint - Short label for the file (e.g. ocr-x85-y0, ocr-trigger)
* @returns Full path of the saved file
*/
Expand All @@ -48,7 +57,7 @@ export async function saveCaptureToDownloads(
const hhmmss = now.toTimeString().slice(0, 8).replace(/:/g, '');
const ms = now.getMilliseconds().toString().padStart(3, '0');
const hint = sanitizeHint(contentHint);
const filename = `qa-auto-hw-${dateStr}-${hhmmss}-${ms}-${hint}.jpg`;
const filename = `qa-auto-hw-${dateStr}-${hhmmss}-${ms}-${hint}.${extensionForDataUrl(dataUrlOrBase64)}`;
const fullPath = path.join(downloadsDir, filename);

const base64 = extractBase64(dataUrlOrBase64);
Expand Down
35 changes: 35 additions & 0 deletions models/paddle_ocr/PP-OCRv6_medium_det_onnx/.gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
Loading
Loading