Fix retry logic: treat OCR ERROR as failure, add retry endpoint and UI button

- worker.py: raise RuntimeError when validate_label returns ERROR status so
  queue.fail() is called and retries are triggered instead of marking the job
  completed
- api.py: add POST /verify/retry/{job_id} endpoint to re-enqueue a failed job
  with the same image and ground truth
- ui_routes.py: add POST /ui/verify/retry/{job_id} route that re-enqueues and
  redirects to the new pending page
- verify_pending.html: show a Retry button in the onFailed() handler that
  POSTs to the retry route

Author: Project Team

app/api.py

@@ -1060,6 +1060,61 @@ async def get_async_verify_status(
     )
 
 
+@app.post("/verify/retry/{job_id}", response_model=AsyncVerifySubmitResponse)
+async def retry_async_verify(
+    job_id: str,
+    username: str = Depends(get_current_user)
+) -> AsyncVerifySubmitResponse:
+    """
+    Re-enqueue a failed (or completed) single-image verify job using the
+    same image and ground truth as the original submission.
+
+    Intended for use after a job reaches ``failed`` status so the user can
+    retry without re-uploading the image.
+
+    **Response:**
+    - ``job_id``: New job identifier — poll ``GET /verify/status/{job_id}``
+    - ``status``: ``pending``
+
+    **Example:**
+    ```bash
+    curl -X POST https://example.com/verify/retry/abc123
+    ```
+    """
+    correlation_id = get_correlation_id()
+    logger.info(f"[{correlation_id}] POST /verify/retry/{job_id}")
+
+    original = verify_queue.get(job_id)
+    if original is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Verify job {job_id} not found",
+        )
+
+    # Make sure the image file still exists on the shared volume
+    image_path = original.get("image_path")
+    if not image_path or not Path(image_path).exists():
+        raise HTTPException(
+            status_code=status.HTTP_410_GONE,
+            detail="Original image file no longer available; please re-upload.",
+        )
+
+    new_job_id = verify_queue.enqueue(
+        image_path=image_path,
+        ground_truth=original.get("ground_truth"),
+    )
+
+    logger.info(
+        f"[{correlation_id}] Retried job {job_id} as new job {new_job_id}"
+    )
+
+    return AsyncVerifySubmitResponse(
+        job_id=new_job_id,
+        status="pending",
+        message=f"Job re-submitted. Poll GET /verify/status/{new_job_id} for results.",
+    )
+
+
 # ============================================================================
 # Exception Handlers
 # ============================================================================

app/templates/verify_pending.html

@@ -69,6 +69,12 @@ <h4 class="mb-2" id="statusHeading">Verifying Label</h4>
 
     <!-- Action buttons (hidden until terminal state) -->
     <div id="actions" style="display:none;" class="mt-3 d-grid gap-2">
+        <!-- Retry button: only shown on failure; submits a form POST to re-enqueue -->
+        <form id="retryForm" method="post" style="display:none;">
+            <button type="submit" class="btn btn-warning w-100">
+                <i class="bi bi-arrow-clockwise"></i> Retry
+            </button>
+        </form>
         <a href="/ui/verify" class="btn btn-primary">
             <i class="bi bi-arrow-repeat"></i> Verify Another Label
         </a>
@@ -96,6 +102,7 @@ <h4 class="mb-2" id="statusHeading">Verifying Label</h4>
     const elError    = document.getElementById('errorDetail');
     const elErrorTxt = document.getElementById('errorText');
     const elActions  = document.getElementById('actions');
+    const elRetryForm = document.getElementById('retryForm');
 
     let pollTimer = null;
 
@@ -171,6 +178,11 @@ <h4 class="mb-2" id="statusHeading">Verifying Label</h4>
         updateDots(data.attempts, data.max_attempts, true);
         elErrorTxt.textContent = data.error || 'Unknown error';
         elError.style.display = 'block';
+
+        // Wire up the retry form to POST to the UI retry endpoint
+        elRetryForm.action = `/ui/verify/retry/${JOB_ID}`;
+        elRetryForm.style.display = 'block';
+
         elActions.style.display = 'block';
     }
 

app/ui_routes.py

@@ -502,6 +502,56 @@ async def ui_verify_result(
     )
 
 
+@router.post("/ui/verify/retry/{job_id}")
+async def ui_verify_retry(
+    request: Request,
+    job_id: str,
+    username: str = Depends(get_current_user_ui)
+):
+    """
+    Re-enqueue a failed verify job from the UI and redirect to the new pending page.
+    Called by the Retry button on the verify_pending page.
+    """
+    from api import verify_queue
+
+    original = verify_queue.get(job_id)
+    if original is None:
+        return templates.TemplateResponse(
+            "index.html",
+            {
+                "request": request,
+                "username": username,
+                "error": "Job not found or expired. Please re-upload your image.",
+                "ollama_host": settings.ollama_host,
+                "default_timeout": settings.ollama_timeout_seconds,
+            },
+        )
+
+    image_path = original.get("image_path")
+    if not image_path or not Path(image_path).exists():
+        return templates.TemplateResponse(
+            "index.html",
+            {
+                "request": request,
+                "username": username,
+                "error": "Original image file is no longer available. Please re-upload.",
+                "ollama_host": settings.ollama_host,
+                "default_timeout": settings.ollama_timeout_seconds,
+            },
+        )
+
+    new_job_id = verify_queue.enqueue(
+        image_path=image_path,
+        ground_truth=original.get("ground_truth"),
+    )
+    logger.info(f"[ui] Retried job {job_id} as new job {new_job_id}")
+
+    return RedirectResponse(
+        url=f"/ui/verify/pending/{new_job_id}",
+        status_code=status.HTTP_303_SEE_OTHER,
+    )
+
+
 @router.get("/ui/health", response_class=HTMLResponse)
 async def ui_health(request: Request):
     """

app/worker.py

@@ -90,6 +90,14 @@ def process_job(job: dict, validator: LabelValidator) -> dict:
 
     result = validator.validate_label(image_path, ground_truth)
     result["image_path"] = Path(image_path).name
+
+    # If the validator returned an ERROR status (e.g. Ollama sentinel absent,
+    # OCR extraction failed), treat it as a retriable failure rather than a
+    # completed job.  Raising here causes the worker loop to call queue.fail(),
+    # which requeues the job if attempts remain.
+    if result.get("status") == "ERROR":
+        raise RuntimeError(result.get("error") or "OCR returned ERROR status")
+
     return result