feat: resilient background job retry & monitoring

app/src/App.tsx

@@ -16,6 +16,7 @@ import NotFound from "./pages/NotFound";
 import { Landing } from "./pages/Landing";
 import ProtectedRoute from "./components/auth/ProtectedRoute";
 import Account from "./pages/Account";
+import Jobs from "./pages/Jobs";
 
 const queryClient = new QueryClient({
   defaultOptions: {
@@ -83,6 +84,14 @@ const App = () => (
                 </ProtectedRoute>
               }
             />
+            <Route
+              path="jobs"
+              element={
+                <ProtectedRoute>
+                  <Jobs />
+                </ProtectedRoute>
+              }
+            />
             <Route
               path="account"
               element={

app/src/api/jobs.ts

@@ -0,0 +1,43 @@
+import { api } from './client';
+
+export type JobStatus = 'PENDING' | 'RUNNING' | 'SUCCESS' | 'FAILED' | 'RETRYING' | 'DEAD';
+
+export type Job = {
+  id: number;
+  name: string;
+  status: JobStatus;
+  attempts: number;
+  max_retries: number;
+  last_error: string | null;
+  result: string | null;
+  scheduled_at: string | null;
+  started_at: string | null;
+  completed_at: string | null;
+  created_at: string;
+};
+
+export type JobStats = {
+  total: number;
+  by_status: Record<string, number>;
+  success_rate: number;
+};
+
+export async function getJobStats(): Promise<JobStats> {
+  return api<JobStats>('/jobs/stats');
+}
+
+export async function getRecentJobs(limit = 20): Promise<Job[]> {
+  return api<Job[]>(`/jobs/recent?limit=${limit}`);
+}
+
+export async function getDeadLetterJobs(limit = 50): Promise<Job[]> {
+  return api<Job[]>(`/jobs/dead-letter?limit=${limit}`);
+}
+
+export async function getJob(id: number): Promise<Job> {
+  return api<Job>(`/jobs/${id}`);
+}
+
+export async function retryJob(id: number): Promise<Job> {
+  return api<Job>(`/jobs/${id}/retry`, { method: 'POST' });
+}

app/src/components/layout/Navbar.tsx

@@ -13,6 +13,7 @@ const navigation = [
   { name: 'Reminders', href: '/reminders' },
   { name: 'Expenses', href: '/expenses' },
   { name: 'Analytics', href: '/analytics' },
+  { name: 'Jobs', href: '/jobs' },
 ];
 
 export function Navbar() {

app/src/pages/Jobs.tsx

@@ -0,0 +1,174 @@
+import { useEffect, useState } from 'react';
+import { Button } from '@/components/ui/button';
+import { useToast } from '@/components/ui/use-toast';
+import { Activity, RefreshCw, AlertTriangle, CheckCircle, Clock, XCircle, RotateCcw } from 'lucide-react';
+import {
+  getJobStats,
+  getRecentJobs,
+  getDeadLetterJobs,
+  retryJob,
+  type Job,
+  type JobStats,
+} from '@/api/jobs';
+
+const STATUS_CONFIG: Record<string, { icon: typeof Activity; color: string; bg: string }> = {
+  PENDING: { icon: Clock, color: 'text-yellow-600', bg: 'bg-yellow-100' },
+  RUNNING: { icon: Activity, color: 'text-blue-600', bg: 'bg-blue-100' },
+  SUCCESS: { icon: CheckCircle, color: 'text-green-600', bg: 'bg-green-100' },
+  FAILED: { icon: XCircle, color: 'text-red-600', bg: 'bg-red-100' },
+  RETRYING: { icon: RefreshCw, color: 'text-orange-600', bg: 'bg-orange-100' },
+  DEAD: { icon: AlertTriangle, color: 'text-red-800', bg: 'bg-red-200' },
+};
+
+export default function Jobs() {
+  const [stats, setStats] = useState<JobStats | null>(null);
+  const [recentJobs, setRecentJobs] = useState<Job[]>([]);
+  const [deadJobs, setDeadJobs] = useState<Job[]>([]);
+  const [tab, setTab] = useState<'recent' | 'dead'>('recent');
+  const [loading, setLoading] = useState(true);
+  const { toast } = useToast();
+
+  const fetchData = async () => {
+    try {
+      const [s, r, d] = await Promise.all([
+        getJobStats(),
+        getRecentJobs(),
+        getDeadLetterJobs(),
+      ]);
+      setStats(s);
+      setRecentJobs(r);
+      setDeadJobs(d);
+    } catch {
+      toast({ title: 'Error', description: 'Failed to load job data', variant: 'destructive' });
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  useEffect(() => { fetchData(); }, []);
+
+  const handleRetry = async (jobId: number) => {
+    try {
+      await retryJob(jobId);
+      toast({ title: 'Success', description: 'Job queued for retry' });
+      fetchData();
+    } catch {
+      toast({ title: 'Error', description: 'Failed to retry job', variant: 'destructive' });
+    }
+  };
+
+  if (loading) {
+    return (
+      <div className="flex items-center justify-center min-h-[400px]">
+        <div className="animate-spin rounded-full h-8 w-8 border-b-2 border-primary" />
+      </div>
+    );
+  }
+
+  const displayJobs = tab === 'recent' ? recentJobs : deadJobs;
+
+  return (
+    <div className="max-w-5xl mx-auto p-6 space-y-6">
+      <div className="flex items-center justify-between">
+        <div className="flex items-center gap-3">
+          <Activity className="h-8 w-8 text-primary" />
+          <h1 className="text-2xl font-bold">Job Monitor</h1>
+        </div>
+        <Button variant="outline" onClick={fetchData}>
+          <RefreshCw className="h-4 w-4 mr-2" />
+          Refresh
+        </Button>
+      </div>
+
+      {/* Stats Cards */}
+      {stats && (
+        <div className="grid grid-cols-2 md:grid-cols-4 gap-4">
+          <div className="border rounded-lg p-4 bg-card">
+            <p className="text-sm text-muted-foreground">Total Jobs</p>
+            <p className="text-2xl font-bold">{stats.total}</p>
+          </div>
+          <div className="border rounded-lg p-4 bg-card">
+            <p className="text-sm text-muted-foreground">Success Rate</p>
+            <p className="text-2xl font-bold text-green-600">{stats.success_rate}%</p>
+          </div>
+          <div className="border rounded-lg p-4 bg-card">
+            <p className="text-sm text-muted-foreground">Failed</p>
+            <p className="text-2xl font-bold text-red-600">
+              {(stats.by_status.FAILED || 0) + (stats.by_status.DEAD || 0)}
+            </p>
+          </div>
+          <div className="border rounded-lg p-4 bg-card">
+            <p className="text-sm text-muted-foreground">Running</p>
+            <p className="text-2xl font-bold text-blue-600">
+              {(stats.by_status.RUNNING || 0) + (stats.by_status.RETRYING || 0)}
+            </p>
+          </div>
+        </div>
+      )}
+
+      {/* Tabs */}
+      <div className="flex gap-2 border-b pb-2">
+        <Button
+          variant={tab === 'recent' ? 'default' : 'ghost'}
+          size="sm"
+          onClick={() => setTab('recent')}
+        >
+          Recent Jobs ({recentJobs.length})
+        </Button>
+        <Button
+          variant={tab === 'dead' ? 'default' : 'ghost'}
+          size="sm"
+          onClick={() => setTab('dead')}
+        >
+          <AlertTriangle className="h-4 w-4 mr-1" />
+          Dead Letter ({deadJobs.length})
+        </Button>
+      </div>
+
+      {/* Job List */}
+      {displayJobs.length === 0 ? (
+        <div className="text-center py-12 text-muted-foreground">
+          <Activity className="h-12 w-12 mx-auto mb-4 opacity-50" />
+          <p className="text-lg">No {tab === 'dead' ? 'dead letter' : 'recent'} jobs</p>
+        </div>
+      ) : (
+        <div className="space-y-2">
+          {displayJobs.map((job) => {
+            const config = STATUS_CONFIG[job.status] || STATUS_CONFIG.PENDING;
+            const Icon = config.icon;
+            return (
+              <div key={job.id} className="border rounded-lg p-4 flex items-center justify-between bg-card">
+                <div className="flex items-center gap-3">
+                  <div className={`p-2 rounded-full ${config.bg}`}>
+                    <Icon className={`h-4 w-4 ${config.color}`} />
+                  </div>
+                  <div>
+                    <p className="font-medium">{job.name}</p>
+                    <p className="text-xs text-muted-foreground">
+                      ID: {job.id} · Attempts: {job.attempts}/{job.max_retries + 1}
+                      {job.completed_at && ` · Completed: ${new Date(job.completed_at).toLocaleString()}`}
+                    </p>
+                    {job.last_error && (
+                      <p className="text-xs text-red-500 mt-1">{job.last_error}</p>
+                    )}
+                  </div>
+                </div>
+                <div className="flex items-center gap-2">
+                  <span className={`text-xs font-medium px-2 py-1 rounded-full ${config.bg} ${config.color}`}>
+                    {job.status}
+                  </span>
+                  {job.status === 'DEAD' && (
+                    <Button size="sm" variant="outline" onClick={() => handleRetry(job.id)}>
+                      <RotateCcw className="h-3 w-3 mr-1" />
+                      Retry
+                    </Button>
+                  )}
+                </div>
+              </div>
+            );
+          })}
+        </div>
+      )}
+    </div>
+  );
+}

packages/backend/app/models.py

@@ -127,6 +127,30 @@ class UserSubscription(db.Model):
     started_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False)
 
 
+class JobStatus(str, Enum):
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    SUCCESS = "SUCCESS"
+    FAILED = "FAILED"
+    RETRYING = "RETRYING"
+    DEAD = "DEAD"
+
+
+class BackgroundJob(db.Model):
+    __tablename__ = "background_jobs"
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(200), nullable=False)
+    status = db.Column(SAEnum(JobStatus), default=JobStatus.PENDING, nullable=False)
+    attempts = db.Column(db.Integer, default=0, nullable=False)
+    max_retries = db.Column(db.Integer, default=3, nullable=False)
+    last_error = db.Column(db.Text, nullable=True)
+    result = db.Column(db.Text, nullable=True)
+    scheduled_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False)
+    started_at = db.Column(db.DateTime, nullable=True)
+    completed_at = db.Column(db.DateTime, nullable=True)
+    created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False)
+
+
 class AuditLog(db.Model):
     __tablename__ = "audit_logs"
     id = db.Column(db.Integer, primary_key=True)

packages/backend/app/routes/__init__.py

@@ -7,6 +7,7 @@
 from .categories import bp as categories_bp
 from .docs import bp as docs_bp
 from .dashboard import bp as dashboard_bp
+from .jobs import bp as jobs_bp
 
 
 def register_routes(app: Flask):
@@ -18,3 +19,4 @@ def register_routes(app: Flask):
     app.register_blueprint(categories_bp, url_prefix="/categories")
     app.register_blueprint(docs_bp, url_prefix="/docs")
     app.register_blueprint(dashboard_bp, url_prefix="/dashboard")
+    app.register_blueprint(jobs_bp, url_prefix="/jobs")

packages/backend/app/routes/jobs.py

@@ -0,0 +1,67 @@
+from flask import Blueprint, jsonify, request
+from flask_jwt_extended import jwt_required, get_jwt_identity
+from ..services.job_runner import (
+    get_job_stats,
+    get_recent_jobs,
+    get_dead_letter_jobs,
+    retry_dead_job,
+)
+from ..models import BackgroundJob, JobStatus
+from ..extensions import db
+import logging
+
+bp = Blueprint("jobs", __name__)
+logger = logging.getLogger("finmind.jobs")
+
+
+@bp.get("/stats")
+@jwt_required()
+def job_stats():
+    stats = get_job_stats()
+    return jsonify(stats)
+
+
+@bp.get("/recent")
+@jwt_required()
+def recent_jobs():
+    limit = request.args.get("limit", 20, type=int)
+    jobs = get_recent_jobs(limit=min(limit, 100))
+    return jsonify(jobs)
+
+
+@bp.get("/dead-letter")
+@jwt_required()
+def dead_letter_jobs():
+    limit = request.args.get("limit", 50, type=int)
+    jobs = get_dead_letter_jobs(limit=min(limit, 100))
+    return jsonify(jobs)
+
+
+@bp.get("/<int:job_id>")
+@jwt_required()
+def get_job(job_id: int):
+    job = db.session.get(BackgroundJob, job_id)
+    if not job:
+        return jsonify(error="not found"), 404
+    return jsonify({
+        "id": job.id,
+        "name": job.name,
+        "status": job.status.value,
+        "attempts": job.attempts,
+        "max_retries": job.max_retries,
+        "last_error": job.last_error,
+        "result": job.result,
+        "scheduled_at": job.scheduled_at.isoformat() if job.scheduled_at else None,
+        "started_at": job.started_at.isoformat() if job.started_at else None,
+        "completed_at": job.completed_at.isoformat() if job.completed_at else None,
+        "created_at": job.created_at.isoformat(),
+    })
+
+
+@bp.post("/<int:job_id>/retry")
+@jwt_required()
+def retry_job(job_id: int):
+    result = retry_dead_job(job_id)
+    if not result:
+        return jsonify(error="Job not found or not in dead state"), 404
+    return jsonify(result)