feat(Jornal da Paraiba): Criação do spider e scraper do portal de notícias Jornal da Paraiba

Dockerfile

@@ -13,6 +13,7 @@ RUN /root/.local/bin/pipenv sync --dev
 # Instala minio e browsers no venv
 RUN /root/.local/bin/pipenv install minio==7.2.15
 RUN /root/.local/bin/pipenv install scrapy-playwright==0.0.43
+ENV PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers
 RUN /root/.local/bin/pipenv run playwright install --with-deps firefox
 
 # Confirma que playwright foi instalado corretamente
@@ -34,7 +35,8 @@ RUN apt-get update \
 
 ENV PATH=/usr/src/.venv/bin:$PATH \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONUNBUFFERED=1
+    PYTHONUNBUFFERED=1 \
+    PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers
 
 WORKDIR /project
 

Makefile

@@ -82,6 +82,10 @@ scrape_uol:
 
 scrape_folha:
 	docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br
+
+scrape_jornaldaparaiba:
+	docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br
+
 
 # Crawler para todos os portais ou específicos
 crawl:
@@ -112,6 +116,9 @@ crawl_ig:
 crawl_folha:
 	docker compose run scraper python crawl.py folhaspider
 
+crawl_jornaldaparaiba:
+	docker compose run scraper python crawl.py jornaldaparaiba
+
 # Workflow completo de coleta de URLs
 crawl_all_working:
 	@echo "Executando crawl de todos os portais funcionais..."
@@ -123,6 +130,7 @@ crawl_all_working:
 	@make crawl_aliadosBrasil
 	@make crawl_ig
 	@make crawl_folha
+	@make crawl_jornaldaparaiba
 	@echo "Crawl de todos os portais concluído!"
 
 # Workflow completo de scraping
@@ -136,6 +144,7 @@ scrape_all_working:
 	@make scrape_r7
 	@make scrape_uol
 	@make scrape_folha
+	@make scrape_jornaldaparaiba
 	@echo "Scraping de todos os portais concluído!"
 
 # Pipeline completo: crawl + scrape
@@ -219,25 +228,24 @@ run-frontend:
 
 # Rodar backend (server-web) com Docker
 run-backend-docker:
-	docker-compose up server-web
+	docker compose up server-web
 
 # Rodar frontend (client-web) com Docker
 run-frontend-docker:
-	docker-compose up client-web
+	docker compose up client-web
 
 # Construir e rodar backend (server-web) com Docker
 build-backend-docker:
-	docker-compose up --build server-web
+	docker compose up --build server-web
 
 # Construir e rodar frontend (client-web) com Docker
 build-frontend-docker:
-	docker-compose up --build client-web
+	docker compose up --build client-web
 
 # Rodar todos os serviços (backend, frontend, minio, db, scraper)
 all:
-	docker-compose up -d
+	docker compose up -d
 
 # Parar todos os serviços
 down:
-	docker-compose down
-
+	docker compose down

create_db.py

@@ -42,6 +42,7 @@
         ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"),
         ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"),
         ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"),
+        ("Jornal da Paraíba", "https://jornaldaparaiba.com.br/", "jornaldaparaiba"),
     ]
     portals_to_add = []
     for portal in portals:

plays/__init__.py

@@ -11,6 +11,7 @@
 from plays.gazetaDoPovo import GazetaDoPovoPlay
 from plays.maisGoias import MaisGoias
 from plays.aliadosBrasil import AliadosBrasilPlay
+from plays.jornaldaparaiba import JornalDaParaibaPlay
 
 __all__ = [
     ClicRBSPlay,
@@ -26,4 +27,5 @@
     GazetaDoPovoPlay,
     MaisGoias,
     AliadosBrasilPlay,
+    JornalDaParaibaPlay,
 ]

plays/base.py

@@ -58,7 +58,7 @@ def get_scraper(cls, url, *args, **kwargs):
 
     def get_session_dir(self):
         if self.session_dir is None:
-            return f"./sessions/{self.name}_session/"
+            return f"./sessions/{self.name}_session/"   
 
         return self.session_dir
 

plays/jornaldaparaiba.py

@@ -0,0 +1,141 @@
+import time
+from playwright.sync_api import sync_playwright
+from plays.base import BasePlay
+from plays.items import EntryItem
+from plog import logger
+
+class JornalDaParaibaPlay(BasePlay):
+    name = "jornaldaparaiba"
+
+    @classmethod
+    def match(cls, url):
+        """Identifica se a URL é do Jornal da Paraíba"""
+        return "jornaldaparaiba.com.br" in url
+
+    def pre_run(self):
+        """Configurações antes da execução"""
+        pass
+
+    def run(self) -> EntryItem:
+        """Extrai conteúdo da notícia"""
+        with sync_playwright() as p:
+            browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080})
+            page = browser.new_page()
+
+            logger.info(f"[{self.name}] Opening URL '{self.url}'...")
+            page.goto(self.url, timeout=180_000)
+
+            # Aguardar carregamento
+            page.wait_for_selector("h1.jpb-article-title", timeout=30000)
+
+            # 1. EXTRAIR TÍTULO
+            title = self._extract_title(page)
+
+            # 2. EXTRAIR DESCRIÇÃO
+            description = self._extract_description(page)
+
+            # 3. EXTRAIR CORPO
+            body = self._extract_body(page)
+
+            # 4. EXTRAIR TAGS
+            tags = self._extract_tags(page)
+
+            return EntryItem(
+                title=title,
+                url=self.url,
+                description=description,
+                body=body,
+                tags=tags,
+            )
+
+    def _extract_title(self, page):
+        """Extrai o título da notícia"""
+        selectors = [
+            "h1.jpb-article-title",
+        ]
+
+        for selector in selectors:
+            try:
+                element = page.locator(selector)
+                if element.count() > 0:
+                    title = element.first.inner_text().strip()
+                    if title:
+                        logger.info(f"[{self.name}] Title extracted: {title[:50]}...")
+                        return title
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract title with {selector}: {str(e)}")
+                continue
+
+        logger.error(f"[{self.name}] Failed to extract title")
+        return ""
+
+    def _extract_description(self, page):
+        """Extrai a descrição/subtítulo da notícia"""
+        selectors = [
+            "h2.jpb-article-subtitle",
+        ]
+
+        for selector in selectors:
+            try:
+                element = page.locator(selector)
+                if element.count() > 0:
+                    description = element.first.inner_text().strip()
+                    if description and len(description) > 10:
+                        logger.info(f"[{self.name}] Description extracted: {description[:50]}...")
+                        return description
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract description with {selector}: {str(e)}")
+                continue
+
+        logger.info(f"[{self.name}] No description found")
+        return ""
+
+    def _extract_body(self, page):
+        """Extrai o corpo da notícia"""
+        selectors = [
+            "div#article",
+        ]
+
+        for selector in selectors:
+            try:
+                element = page.locator(selector)
+                if element.count() > 0:
+                    # Remover elementos indesejados
+                    element.locator("script, style, .ad, .advertisement").evaluate_all("els => els.forEach(el => el.remove())")
+
+                    body = element.inner_text().strip()
+                    if body and len(body) > 100:  # Garantir que tem conteúdo substancial
+                        logger.info(f"[{self.name}] Body extracted ({len(body)} chars)")
+                        return body
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract body with {selector}: {str(e)}")
+                continue
+
+        logger.error(f"[{self.name}] Failed to extract body")
+        return ""
+
+    def _extract_tags(self, page):
+        """Extrai as tags da notícia"""
+        selectors = [
+            "section.jpb-tags a",
+        ]
+
+        tags = []
+
+        for selector in selectors:
+            try:
+                elements = page.locator(selector)
+                if elements.count() > 0:
+                    for i in range(elements.count()):
+                        tag_text = elements.nth(i).inner_text().strip()
+                        if tag_text and tag_text not in tags:
+                            tags.append(tag_text)
+
+                    if tags:
+                        logger.info(f"[{self.name}] Tags extracted: {tags}")
+                        break
+            except Exception as e:
+                logger.warning(f"[{self.name}] Failed to extract tags with {selector}: {str(e)}")
+                continue
+
+        return tags

spiders/__init__.py

@@ -12,6 +12,7 @@
 from spiders.gazetaDoPovo import GazetaDoPovoSpider
 from spiders.maisGoias import MaisGoiasSpider
 from spiders.aliadosBrasil import AliadosBrasilSpider
+from spiders.jornaldaparaiba import JornalDaParaibaSpider
 
 __all__ = [
     BaseSpider,
@@ -28,4 +29,5 @@
     GazetaDoPovoSpider,
     MaisGoiasSpider,
     AliadosBrasilSpider,
+    JornalDaParaibaSpider,
 ]

spiders/jornaldaparaiba.py

@@ -0,0 +1,93 @@
+import scrapy
+from urllib.parse import urlparse
+from spiders.base import BaseSpider
+from spiders.items import URLItem
+
+class JornalDaParaibaSpider(BaseSpider):
+    name = "jornaldaparaiba"
+    start_urls = ["https://jornaldaparaiba.com.br/"]
+    allowed_domains = ["jornaldaparaiba.com.br"]
+
+    custom_settings = {
+        **BaseSpider.custom_settings,
+        "COOKIES_ENABLED": True,
+        "DOWNLOAD_DELAY": 2,
+        "DEFAULT_REQUEST_HEADERS": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+        }
+    }
+
+    def allow_url(self, url: str) -> bool:
+        """
+        Filtra apenas URLs de notícias válidas
+        """
+        p = urlparse(url)
+        path = p.path.rstrip('/')
+
+        # Blacklist de seções que não são notícias
+        blacklist = {
+            "/videos", "/podcasts", "/newsletter", "/contato",
+            "/sobre", "/anuncie", "/assine", "/busca", "/expediente",
+            "/jornal-da-paraiba-25-anos", "/colunistas", "/guia-qualaboa",
+            "/vamos-trabalhar", "/educacao", "/espaco-opiniao"
+        }
+
+        if any(blacklisted in path for blacklisted in blacklist):
+            return False
+
+        # Requer pelo menos 2 segmentos (seção + slug)
+        segments = [seg for seg in path.split('/') if seg]
+        if len(segments) < 2:
+            return False
+
+        # Verificar se é uma notícia (slug longo com hífens)
+        slug = segments[-1]
+        if slug.count('-') >= 2 or len(slug) > 20:
+            return True
+
+        return False
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield scrapy.Request(
+                url,
+                callback=self.parse,
+                dont_filter=True,
+                meta={"dont_redirect": True, "handle_httpstatus_list": [403]},
+            )
+
+    def parse(self, response):
+        """
+        Extrai URLs de notícias da página inicial
+        """
+        selectors = [
+            "section.jpb-chamadaBoxHorizontal a[href]",
+            "section.jpb-maisLidas a[href]",
+            "section.jpb-chamadaBox a[href]",
+            "section.jpb-ultimasNoticias a[href]",
+        ]
+
+        seen_urls = set()
+
+        for selector in selectors:
+            links = response.css(selector)
+
+            for link in links:
+                url = link.attrib.get("href")
+                if not url:
+                    continue
+
+                # Construir URL absoluta
+                full_url = response.urljoin(url)
+                full_url = full_url.split('#', 1)[0].split('?', 1)[0]
+
+                # Evitar duplicatas
+                if full_url in seen_urls:
+                    continue
+                seen_urls.add(full_url)
+
+                # Filtrar URLs válidos
+                if self.allow_url(full_url):
+                    yield URLItem(url=full_url)