Feat/parmanent scraping task

docker-compose.yml

@@ -1,6 +1,5 @@
 services:
   frontend:
-    user: node
 
     build:
       context: ./frontend
@@ -10,44 +9,47 @@ services:
       - NODE_ENV=production
     networks:
       - rishu-app_network
+    user: node
 
   web:
     build:
       context: ./nginx
       dockerfile: Dockerfile
     restart: always
+    depends_on:
+      - frontend
+      - syllabus-backend
     ports:
       - 80:80
     environment:
       - SERVER_NAME=${SERVER_NAME}
     networks:
       - rishu-app_network
-    depends_on:
-      - frontend
-      - syllabus-backend
 
   syllabus-backend:
     build:
       context: ./syllabus-backend
       dockerfile: Dockerfile
     restart: always
+    depends_on:
+      - syllabus-db
+    ports:
+      - 8080:8080
     environment:
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       POSTGRES_DB: ${POSTGRES_DB}
     networks:
       - rishu-app_network
-    depends_on:
-      - syllabus-db
 
   syllabus-db:
     image: postgres:17.4-bullseye
+    volumes:
+      - syllabus-db:/var/lib/postgresql/data
     environment:
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       POSTGRES_DB: ${POSTGRES_DB}
-    volumes:
-      - syllabus-db:/var/lib/postgresql/data
     networks:
       - rishu-app_network
 
@@ -56,18 +58,33 @@ services:
       context: ./syllabus-scrape
       dockerfile: Dockerfile
     restart: always
+    depends_on:
+      - syllabus-scrape-redis
+      - syllabus-backend
     environment:
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       POSTGRES_DB: ${POSTGRES_DB}
       NODE_ENV: production
       SYLLABUS_BACKEND_URL: http://syllabus-backend:8080
+      REDIS_HOST: syllabus-scrape-redis
+      REDIS_PORT: 6379
     networks:
       - rishu-app_network
 
-networks:
-  rishu-app_network:
-    driver: bridge
+  syllabus-scrape-redis:
+    image: redis:8.0-alpine
+    restart: always
+    volumes:
+      - redis-data:/data
+    networks:
+      - rishu-app_network
 
 volumes:
   syllabus-db:
+  redis-data:
+
+
+networks:
+  rishu-app_network:
+    driver: bridge

syllabus-scrape/package.json

@@ -23,6 +23,7 @@
     "typescript": "^5.9.2"
   },
   "dependencies": {
+    "bullmq": "^5.74.1",
     "cheerio": "^1.1.2",
     "puppeteer": "^24.16.2"
   }