feat(Jornal de Brasília): Criação do scraper completo do Jornal de Brasília

Makefile

@@ -82,7 +82,10 @@ scrape_uol:
 
 scrape_folha:
 	docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br
-
+
+scrape_jornalDeBrasilia:
+	docker compose exec scraper python scrape_no_openai.py --platform jornaldebrasilia.com.br
+
 # Crawler para todos os portais ou específicos
 crawl:
 	docker compose run --rm scraper python crawl.py
@@ -112,6 +115,9 @@ crawl_ig:
 crawl_folha:
 	docker compose run scraper python crawl.py folhaspider
 
+crawl_jornalDeBrasilia: 
+	docker compose run scraper python crawl.py jornaldebrasiliaspider
+
 # Workflow completo de coleta de URLs
 crawl_all_working:
 	@echo "Executando crawl de todos os portais funcionais..."
@@ -123,6 +129,7 @@ crawl_all_working:
 	@make crawl_aliadosBrasil
 	@make crawl_ig
 	@make crawl_folha
+	@make crawl_jornalDeBrasilia
 	@echo "Crawl de todos os portais concluído!"
 
 # Workflow completo de scraping
@@ -136,6 +143,7 @@ scrape_all_working:
 	@make scrape_r7
 	@make scrape_uol
 	@make scrape_folha
+	@make scrape_jornalDeBrasilia
 	@echo "Scraping de todos os portais concluído!"
 
 # Pipeline completo: crawl + scrape

create_db.py

@@ -42,6 +42,7 @@
         ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"),
         ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"),
         ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"),
+        ("Jornal de Brasília", "https://www.jornaldebrasilia.com.br/", "jornaldebrasilia"),
     ]
     portals_to_add = []
     for portal in portals:

plays/__init__.py

@@ -11,6 +11,7 @@
 from plays.gazetaDoPovo import GazetaDoPovoPlay
 from plays.maisGoias import MaisGoias
 from plays.aliadosBrasil import AliadosBrasilPlay
+from plays.jornalDeBrasilia import JornalDeBrasiliaPlay
 
 __all__ = [
     ClicRBSPlay,
@@ -26,4 +27,5 @@
     GazetaDoPovoPlay,
     MaisGoias,
     AliadosBrasilPlay,
+    JornalDeBrasiliaPlay,
 ]

plays/jornalDeBrasilia.py

@@ -0,0 +1,151 @@
+from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
+
+from plays.base import BasePlay
+from plays.items import EntryItem
+from plog import logger
+
+
+class JornalDeBrasiliaPlay(BasePlay):
+    name = "jornaldebrasilia"
+
+    @classmethod
+    def match(cls, url: str) -> bool:
+        """Verifica se a URL pertence ao Jornal de Brasília."""
+        return "jornaldebrasilia.com.br" in url
+
+    def pre_run(self):
+        """Método executado antes do 'run'."""
+        logger.info(f"[{self.name}] Preparando para extrair a URL: {self.url}")
+
+    def run(self) -> EntryItem:
+        """Executa a extração principal dos dados da página."""
+        with sync_playwright() as p:
+            browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080})
+            page = browser.new_page(
+            )
+
+            page.route(
+                "**/*",
+                lambda route: route.abort()
+                if route.request.resource_type in {"image", "font", "media", "stylesheet"}
+                else route.continue_(),
+            )
+
+            page.set_default_navigation_timeout(60_000)
+            page.set_default_timeout(10_000) 
+
+            try:
+                page.goto(self.url, wait_until="domcontentloaded", timeout=60_000)
+                page.wait_for_timeout(2000) 
+            except PlaywrightTimeoutError:
+                logger.error(f"[{self.name}] Timeout ao carregar a página: {self.url}")
+                browser.close()
+                return EntryItem(title="", url=self.url, description="", body="", tags=[])
+
+
+            # Título
+            title = ""
+            title_selectors = [
+                "h1.tdb-title-text",
+                ".wrap-title > h1",
+                "h1.title",
+                "h1.entry-title",
+                ".post-title",
+                ".content-head__title",
+                "h1" 
+            ]
+            for selector in title_selectors:
+                try:
+                    el = page.locator(selector).first
+                    if el.is_visible(timeout=1000):
+                        title = el.inner_text().strip()
+                        if title:
+                            logger.info(f"[{self.name}] Título encontrado com o seletor: '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+            if not title:
+                try:
+                    title = page.title().strip().split("|")[0].strip()
+                except Exception:
+                    title = ""
+
+
+            # Descrição (Subtítulo)
+            description = ""
+            description_selectors = [
+                ".wrap-title > p:nth-child(2)",
+                "p.tdb-post-sub-title",
+                ".subtitle h2",
+                "p.td-post-sub-title"
+            ]
+            for selector in description_selectors:
+                try:
+                    el = page.locator(selector).first
+                    if el.is_visible(timeout=1000):
+                        description = el.inner_text().strip()
+                        if description:
+                            logger.info(f"[{self.name}] Descrição encontrada com o seletor: '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+            # Corpo da Notícia
+            body = ""
+            body_container_selectors = [
+                "div.tdb-block-inner.td-fix-index",
+                ".the-post-content",
+                ".td-post-content"
+            ]
+            for selector in body_container_selectors:
+                try:
+                    container = page.locator(selector).first
+                    if container.is_visible(timeout=1500):
+                        logger.info(f"[{self.name}] Contêiner do corpo encontrado com o seletor: '{selector}'")
+
+                        # Extrai todos os parágrafos <p> de dentro do contêiner
+                        paragraphs = container.locator("p")
+                        parts = []
+                        for i in range(paragraphs.count()):
+                            p_text = paragraphs.nth(i).inner_text().strip()
+                            # Filtro para remover parágrafos indesejados
+                            if p_text and len(p_text) > 40 and not p_text.lower().startswith(("leia mais", ">>", "crédito:", "siga o jornal")):
+                                parts.append(p_text)
+
+                        if parts:
+                            body = "\n\n".join(parts)
+                            logger.info(f"[{self.name}] Corpo extraído com sucesso! Caracteres: {len(body)}")
+                            break # Sai do loop principal se o corpo foi extraído
+                except Exception:
+                    continue
+
+            # Extração de Tags (opcional, pois nem sempre estão presentes)
+            tags = []
+            tag_selectors = [
+                ".tdb-tags a",
+                ".entities__list-item a",
+                ".entities__list li a"
+            ]
+            for selector in tag_selectors:
+                try:
+                    tag_elements = page.locator(selector)
+                    if tag_elements.count() > 0:
+                        tag_list = [el.inner_text().strip() for el in tag_elements.all()]
+                        tags = [t for t in tag_list if t] # Remove tags vazias
+                        if tags:
+                            logger.info(f"[{self.name}] {len(tags)} tags encontradas com o seletor: '{selector}'")
+                            break
+                except Exception:
+                    continue
+
+            browser.close()
+
+            return EntryItem(
+                title=title,
+                url=self.url,
+                description=description,
+                body=body,
+                tags=tags,
+            )
+

spiders/__init__.py

@@ -1,19 +1,20 @@
+from spiders.aliadosBrasil import AliadosBrasilSpider
 from spiders.base import BaseSpider
 from spiders.estadao import EstadaoSpider
 from spiders.folha import FolhaSpider
+from spiders.gazetaDoPovo import GazetaDoPovoSpider
 from spiders.globo import GloboSpider
 from spiders.ig import IGSpider
+from spiders.jornalDeBrasilia import JornalDeBrasiliaSpider
+from spiders.maisGoias import MaisGoiasSpider
 from spiders.metropoles import MetropolesSpider
 from spiders.r7 import R7Spider
 from spiders.rbs import RBSSpider
-from spiders.veja import VejaSpider
 from spiders.terra import TerraSpider
 from spiders.uol import UOLSpider
-from spiders.gazetaDoPovo import GazetaDoPovoSpider
-from spiders.maisGoias import MaisGoiasSpider
-from spiders.aliadosBrasil import AliadosBrasilSpider
+from spiders.veja import VejaSpider
 
-__all__ = [
+_all_ = [
     BaseSpider,
     EstadaoSpider,
     FolhaSpider,
@@ -28,4 +29,5 @@
     GazetaDoPovoSpider,
     MaisGoiasSpider,
     AliadosBrasilSpider,
-]
+    JornalDeBrasiliaSpider,
+]

spiders/jornalDeBrasilia.py

@@ -0,0 +1,91 @@
+import scrapy
+from urllib.parse import urlparse
+from spiders.base import BaseSpider
+from spiders.items import URLItem
+
+class JornalDeBrasiliaSpider(BaseSpider):
+    name = "jornaldebrasiliaspider"
+    start_urls = ["https://jornaldebrasilia.com.br/"]
+    allowed_domains = ["jornaldebrasilia.com.br"]
+
+    custom_settings = {
+        **BaseSpider.custom_settings,
+        "COOKIES_ENABLED": True,
+        "DOWNLOAD_DELAY": 2,
+        "DEFAULT_REQUEST_HEADERS": {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+            "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+        }
+    }
+
+    def allow_url(self, url: str) -> bool:
+        """
+        Filtra apenas URLs de notícias válidas
+        """
+        p = urlparse(url)
+        path = p.path.rstrip('/')
+
+        # Blacklist de seções que não são notícias
+        blacklist = {
+            "/videos", "/podcasts", "/newsletter", "/contato", 
+            "/sobre", "/anuncie", "/assine", "/busca"
+        }
+
+        if path in blacklist:
+            return False
+
+        # Requer pelo menos 2 segmentos (seção + slug)
+        segments = [seg for seg in path.split('/') if seg]
+        if len(segments) < 2:
+            return False
+
+        # Verificar se é uma notícia (slug longo com hífens)
+        slug = segments[-1]
+        if slug.count('-') >= 2 or len(slug) > 20:
+            return True
+
+        return False
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield scrapy.Request(
+                url,
+                callback=self.parse,
+                dont_filter=True,
+                meta={"dont_redirect": True, "handle_httpstatus_list": [403]},
+            )
+
+    def parse(self, response):
+        """
+        Extrai URLs de notícias da página inicial
+        """
+        selectors = [
+            "article a[href]", 
+            ".news-item a[href]", 
+            ".headline a[href]",  
+            "h2 a[href], h3 a[href]",  
+        ]
+
+        seen_urls = set()
+
+        for selector in selectors:
+            links = response.css(selector)
+
+            for link in links:
+                url = link.attrib.get("href")
+                if not url:
+                    continue
+
+                # Construir URL absoluta
+                full_url = response.urljoin(url)
+                full_url = full_url.split('#', 1)[0].split('?', 1)[0]
+
+                # Evitar duplicatas
+                if full_url in seen_urls:
+                    continue
+                seen_urls.add(full_url)
+
+                # Filtrar URLs válidos
+                if self.allow_url(full_url):
+                    yield URLItem(url=full_url)