fix: implement exponential backoff and throttled scanning to resolve 403 rate limits

GEMINI.md

@@ -83,6 +83,8 @@ This rule has no exceptions.
 
 Always review `GEMINI.md` (and specifically the **Project Conventions** below) before executing any shell commands or pushing code. This ensures strict compliance with local environment constraints (e.g., mandatory PowerShell syntax) and project-specific workflows. This is critical to prevent command failures in the local Windows environment.
 
+**Environment Note:** You are currently in a **local Windows environment**, NOT the production server. Commands related to production process management (e.g., `pm2`) will fail and should not be executed here.
+
 ### Project Conventions
 
 - Sanity Checks: Always run `pnpm lint ; pnpm format ; pnpm typecheck ; pnpm build ; pnpm test --coverage` before committing or creating a Pull Request.

README.md

@@ -41,9 +41,9 @@ ORB is a momentum-based intraday options strategy. After the market opens and se
 
 ## Resilience & Efficiency
 
-- **Batch API Integration:** Uses Angel One's batch market data API to fetch prices for 50+ stocks and options in small chunks, drastically reducing total API calls and avoiding rate-limit throttles.
-- **Exponential Backoff:** Implements robust retry logic with a 2-second delay for failed batch requests, specifically handling `403 Forbidden` WAF rejections.
-- **Diagnostic Observability:** Enhanced logging captures snippets of rejection responses (e.g., broker-side firewall messages) to troubleshoot network-level blocks in production.
+- **Exponential Backoff:** Implements robust retry logic with a 2-second initial delay for failed requests, specifically handling `403 Forbidden` and `429 Too Many Requests` status codes, as well as Angel One's custom "status: false" error patterns.
+- **Throttled Scanning:** The morning scanner uses deliberate delays (3s between stocks, 1s between historical requests) to respect strict broker rate limits (3 requests per second for historical data), ensuring stability during high-load morning periods.
+- **Diagnostic Observability:** Enhanced logging captures snippets of rejection responses and HTTP methods to troubleshoot network-level blocks and rate-limiting in production.
 - **Dynamic Monthly Expiry:** Automatically detects and targets the final Thursday of the current month (or the next, if the current month's expiry has passed) to ensure option chain data is always current and valid.
 
 ## Tech Stack

__tests__/helpers/api.test.ts

@@ -1,10 +1,30 @@
-/* eslint-disable @typescript-eslint/unbound-method */
 import axios, { InternalAxiosRequestConfig } from 'axios';
 import { api } from '../../src/helpers/api.js';
 import { sessionStore } from '../../src/store/sessionStore.js';
 import { logger } from '../../src/helpers/logger.js';
 
-jest.mock('axios');
+// Mock everything before importing api
+jest.mock('axios', () => {
+  const mockClient = {
+    get: jest.fn(),
+    post: jest.fn(),
+    interceptors: {
+      request: { use: jest.fn(), eject: jest.fn() },
+      response: { use: jest.fn(), eject: jest.fn() },
+    },
+  };
+  return {
+    create: jest.fn(() => mockClient),
+    isAxiosError: jest.fn(),
+    // axios default export
+    default: {
+      create: jest.fn(() => mockClient),
+      isAxiosError: jest.fn(),
+      post: jest.fn(),
+      get: jest.fn(),
+    },
+  };
+});
 jest.mock('../../src/helpers/logger.js');
 jest.mock('../../src/store/sessionStore.js');
 
@@ -14,9 +34,16 @@ describe('API Helper', () => {
   let interceptor: (
     config: InternalAxiosRequestConfig,
   ) => InternalAxiosRequestConfig;
+  let mockedClient: {
+    get: jest.Mock;
+    post: jest.Mock;
+    interceptors: { request: { use: jest.Mock } };
+  };
 
   beforeAll(() => {
-    const useMock = mockedAxios.interceptors.request.use as jest.Mock;
+    mockedClient = (mockedAxios.create as jest.Mock).mock.results[0]
+      .value as typeof mockedClient;
+    const useMock = mockedClient.interceptors.request.use;
     const firstCall = useMock.mock.calls[0] as unknown[];
     interceptor = firstCall[0] as (
       config: InternalAxiosRequestConfig,
@@ -25,6 +52,13 @@ describe('API Helper', () => {
 
   beforeEach(() => {
     jest.clearAllMocks();
+    jest.useFakeTimers();
+    // Default mock behavior for isAxiosError
+    (axios.isAxiosError as unknown as jest.Mock).mockReturnValue(true);
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
   });
 
   describe('Interceptors', () => {
@@ -55,30 +89,135 @@ describe('API Helper', () => {
   describe('GET', () => {
     it('should perform successful GET request', async () => {
       const mockData = { success: true };
-      mockedAxios.get.mockResolvedValueOnce({ data: mockData });
+      mockedClient.get.mockResolvedValueOnce({ data: mockData });
 
       const result = await api.get('/test-url');
 
-      expect(mockedAxios.get).toHaveBeenCalledWith('/test-url', undefined);
+      expect(mockedClient.get).toHaveBeenCalledWith('/test-url', undefined);
       expect(result).toEqual(mockData);
     });
 
-    it('should throw error and log on GET failure', async () => {
-      const errorMessage = 'Network Error';
-      mockedAxios.get.mockRejectedValueOnce(new Error(errorMessage));
+    it('should retry on 403 error and succeed', async () => {
+      const mockData = { success: true };
+      const error403 = {
+        response: { status: 403 },
+        isAxiosError: true,
+        message: 'Forbidden',
+      };
+      mockedClient.get
+        .mockRejectedValueOnce(error403)
+        .mockResolvedValueOnce({ data: mockData });
+
+      const promise = api.get('/test-url');
+
+      // First retry wait
+      await jest.advanceTimersByTimeAsync(2000);
+      const result = await promise;
+
+      expect(mockedClient.get).toHaveBeenCalledTimes(2);
+      expect(result).toEqual(mockData);
+      expect(logger.warn).toHaveBeenCalledWith(
+        expect.stringContaining('GET /test-url failed with 403. Retrying'),
+      );
+    });
 
-      await expect(api.get('/test-url')).rejects.toThrow(errorMessage);
+    it('should throw error and log after all retries fail', async () => {
+      const error429 = new Error('Too Many Requests');
+      Object.assign(error429, {
+        response: { status: 429 },
+        isAxiosError: true,
+      });
+
+      mockedClient.get.mockRejectedValue(error429);
+
+      const promise = api.get('/test-url');
+
+      // Allow all retry timers to fire while awaiting rejection
+      await Promise.all([
+        jest.runAllTimersAsync(),
+        expect(promise).rejects.toThrow('Too Many Requests'),
+      ]);
+
+      expect(mockedClient.get).toHaveBeenCalledTimes(3);
       expect(logger.error).toHaveBeenCalledWith(
-        `GET /test-url failed: ${errorMessage}`,
+        expect.stringContaining(
+          'GET /test-url failed after 3 attempts: Too Many Requests',
+        ),
       );
     });
 
+    it('should throw error immediately for non-retryable error', async () => {
+      const error500 = new Error('Internal Server Error');
+      Object.assign(error500, {
+        response: { status: 500 },
+        isAxiosError: true,
+      });
+
+      mockedClient.get.mockRejectedValueOnce(error500);
+
+      const promise = api.get('/test-url');
+      // No timers to advance as it should fail immediately
+
+      await expect(promise).rejects.toThrow('Internal Server Error');
+      expect(mockedClient.get).toHaveBeenCalledTimes(1);
+      expect(logger.error).toHaveBeenCalledWith(
+        expect.stringContaining(
+          'GET /test-url failed after 1 attempts: Internal Server Error',
+        ),
+      );
+    });
+
+    it('should retry on Angel One custom error code and succeed', async () => {
+      const mockData = { success: true };
+      const errorRateLimit = {
+        data: {
+          status: false,
+          message: 'Too many requests',
+          errorcode: 'AG8001',
+        },
+      };
+      mockedClient.get
+        .mockResolvedValueOnce(errorRateLimit)
+        .mockResolvedValueOnce({ data: mockData });
+
+      const promise = api.get('/test-url');
+      await jest.runAllTimersAsync();
+      const result = await promise;
+
+      expect(mockedClient.get).toHaveBeenCalledTimes(2);
+      expect(result).toEqual(mockData);
+      expect(logger.warn).toHaveBeenCalledWith(
+        expect.stringContaining('GET /test-url failed with AG8001. Retrying'),
+      );
+    });
+
+    it('should throw on non-retryable Angel One custom error code', async () => {
+      const errorInvalidToken = {
+        data: {
+          status: false,
+          message: 'Invalid Token',
+          errorcode: 'AG8003',
+        },
+      };
+      mockedClient.get.mockResolvedValueOnce(errorInvalidToken);
+
+      const promise = api.get('/test-url');
+      await expect(promise).rejects.toThrow('Invalid Token (AG8003)');
+      expect(mockedClient.get).toHaveBeenCalledTimes(1);
+    });
+
     it('should log string error on GET failure', async () => {
-      mockedAxios.get.mockRejectedValueOnce('Unknown Error');
+      // For string error, isAxiosError should be false
+      (axios.isAxiosError as unknown as jest.Mock).mockReturnValue(false);
+      mockedClient.get.mockRejectedValueOnce('Unknown Error');
+
+      const promise = api.get('/test-url');
 
-      await expect(api.get('/test-url')).rejects.toBe('Unknown Error');
+      await expect(promise).rejects.toBe('Unknown Error');
       expect(logger.error).toHaveBeenCalledWith(
-        `GET /test-url failed: Unknown Error`,
+        expect.stringContaining(
+          'GET /test-url failed after 1 attempts: Unknown Error',
+        ),
       );
     });
   });
@@ -87,34 +226,50 @@ describe('API Helper', () => {
     it('should perform successful POST request', async () => {
       const mockData = { success: true };
       const postBody = { foo: 'bar' };
-      mockedAxios.post.mockResolvedValueOnce({ data: mockData });
+      mockedClient.post.mockResolvedValueOnce({ data: mockData });
 
       const result = await api.post('/test-url', postBody);
 
-      expect(mockedAxios.post).toHaveBeenCalledWith(
+      expect(mockedClient.post).toHaveBeenCalledWith(
         '/test-url',
         postBody,
         undefined,
       );
       expect(result).toEqual(mockData);
     });
 
-    it('should throw error and log on POST failure', async () => {
-      const errorMessage = 'Network Error';
-      mockedAxios.post.mockRejectedValueOnce(new Error(errorMessage));
+    it('should retry on 429 error and succeed', async () => {
+      const mockData = { success: true };
+      const error429 = {
+        response: { status: 429 },
+        isAxiosError: true,
+        message: 'Rate Limited',
+      };
+      mockedClient.post
+        .mockRejectedValueOnce(error429)
+        .mockResolvedValueOnce({ data: mockData });
 
-      await expect(api.post('/test-url', {})).rejects.toThrow(errorMessage);
-      expect(logger.error).toHaveBeenCalledWith(
-        `POST /test-url failed: ${errorMessage}`,
-      );
+      const promise = api.post('/test-url', {});
+      await jest.advanceTimersByTimeAsync(2000);
+      const result = await promise;
+
+      expect(mockedClient.post).toHaveBeenCalledTimes(2);
+      expect(result).toEqual(mockData);
     });
 
-    it('should log string error on POST failure', async () => {
-      mockedAxios.post.mockRejectedValueOnce('Unknown Error');
+    it('should throw error and log on POST failure after retries', async () => {
+      const errorMessage = 'Network Error';
+      const error = new Error(errorMessage);
+      (axios.isAxiosError as unknown as jest.Mock).mockReturnValue(false);
+      mockedClient.post.mockRejectedValue(error);
+
+      const promise = api.post('/test-url', {});
 
-      await expect(api.post('/test-url', {})).rejects.toBe('Unknown Error');
+      await expect(promise).rejects.toThrow(errorMessage);
       expect(logger.error).toHaveBeenCalledWith(
-        `POST /test-url failed: Unknown Error`,
+        expect.stringContaining(
+          'POST /test-url failed after 1 attempts: Network Error',
+        ),
       );
     });
   });

src/helpers/api.ts

@@ -50,31 +50,84 @@ apiClient.interceptors.request.use(axiosConfig => {
 
 export const api = {
   get: async <T>(url: string, axiosConfig?: AxiosRequestConfig): Promise<T> => {
-    try {
-      const response: AxiosResponse<T> = await apiClient.get(url, axiosConfig);
-      return response.data;
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      logger.error(`GET ${url} failed: ${message}`);
-      throw error;
-    }
+    return withRetry(() => apiClient.get(url, axiosConfig), `GET ${url}`);
   },
   post: async <T>(
     url: string,
     data?: unknown,
     axiosConfig?: AxiosRequestConfig,
   ): Promise<T> => {
+    return withRetry(
+      () => apiClient.post(url, data, axiosConfig),
+      `POST ${url}`,
+    );
+  },
+};
+
+interface AngelOneResponse<T> {
+  status: boolean;
+  message: string;
+  errorcode: string;
+  data: T;
+}
+
+async function withRetry<T>(
+  fn: () => Promise<AxiosResponse<T>>,
+  label: string,
+  retries = 3,
+  delay = 2000,
+): Promise<T> {
+  for (let attempt = 1; attempt <= retries; attempt++) {
     try {
-      const response: AxiosResponse<T> = await apiClient.post(
-        url,
-        data,
-        axiosConfig,
-      );
+      const response = await fn();
+
+      // Handle Angel One's "200 OK but error in body" pattern
+      const data = response.data as unknown as AngelOneResponse<T>;
+      if (data && data.status === false && data.errorcode) {
+        // Treat certain error codes as retryable if needed,
+        // but for now, we'll throw and let the retry logic handle status codes.
+        // If it's a 200 OK with status: false, it won't have a statusCode like 403.
+        const errorMessage = `${data.message} (${data.errorcode})`;
+
+        // If it's a rate limit or session error in the body, we might want to retry
+        const retryableErrorCodes = ['AG8001', 'AG8002', 'AM0001']; // Example codes
+        if (retryableErrorCodes.includes(data.errorcode) && attempt < retries) {
+          logger.warn(
+            `${label} failed with ${data.errorcode}. Retrying... (Attempt ${attempt}/${retries})`,
+          );
+          await new Promise(resolve => setTimeout(resolve, delay));
+          delay *= 2;
+          continue;
+        }
+
+        throw new Error(errorMessage);
+      }
+
       return response.data;
     } catch (error) {
+      const isLastAttempt = attempt === retries;
+      const statusCode = axios.isAxiosError(error) ? error.response?.status : 0;
       const message = error instanceof Error ? error.message : String(error);
-      logger.error(`POST ${url} failed: ${message}`);
-      throw error;
+
+      // 403 and 429 are definitely retryable for Angel One
+      const isRetryable =
+        statusCode === 403 ||
+        statusCode === 429 ||
+        statusCode === 502 ||
+        statusCode === 503 ||
+        statusCode === 504;
+
+      if (isLastAttempt || !isRetryable) {
+        logger.error(`${label} failed after ${attempt} attempts: ${message}`);
+        throw error;
+      }
+
+      logger.warn(
+        `${label} failed with ${statusCode || 'error'}. Retrying in ${delay}ms... (Attempt ${attempt}/${retries})`,
+      );
+      await new Promise(resolve => setTimeout(resolve, delay));
+      delay *= 2;
     }
-  },
-};
+  }
+  throw new Error('Retry loop ended unexpectedly');
+}