diff --git a/Makefile b/Makefile index 12663c4..78e70ca 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,10 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br - + +scrape_jornalDeBrasilia: + docker compose exec scraper python scrape_no_openai.py --platform jornaldebrasilia.com.br + # Crawler para todos os portais ou específicos crawl: docker compose run --rm scraper python crawl.py @@ -112,6 +115,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_jornalDeBrasilia: + docker compose run scraper python crawl.py jornaldebrasiliaspider + # Workflow completo de coleta de URLs crawl_all_working: @echo "Executando crawl de todos os portais funcionais..." @@ -123,6 +129,7 @@ crawl_all_working: @make crawl_aliadosBrasil @make crawl_ig @make crawl_folha + @make crawl_jornalDeBrasilia @echo "Crawl de todos os portais concluído!" # Workflow completo de scraping @@ -136,6 +143,7 @@ scrape_all_working: @make scrape_r7 @make scrape_uol @make scrape_folha + @make scrape_jornalDeBrasilia @echo "Scraping de todos os portais concluído!" # Pipeline completo: crawl + scrape diff --git a/create_db.py b/create_db.py index 617b85b..3232ffe 100644 --- a/create_db.py +++ b/create_db.py @@ -42,6 +42,7 @@ ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"), ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"), ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"), + ("Jornal de Brasília", "https://www.jornaldebrasilia.com.br/", "jornaldebrasilia"), ] portals_to_add = [] for portal in portals: diff --git a/plays/__init__.py b/plays/__init__.py index 338fe28..700eb9e 100644 --- a/plays/__init__.py +++ b/plays/__init__.py @@ -11,6 +11,7 @@ from plays.gazetaDoPovo import GazetaDoPovoPlay from plays.maisGoias import MaisGoias from plays.aliadosBrasil import AliadosBrasilPlay +from plays.jornalDeBrasilia import JornalDeBrasiliaPlay __all__ = [ ClicRBSPlay, @@ -26,4 +27,5 @@ GazetaDoPovoPlay, MaisGoias, AliadosBrasilPlay, + JornalDeBrasiliaPlay, ] diff --git a/plays/jornalDeBrasilia.py b/plays/jornalDeBrasilia.py new file mode 100644 index 0000000..c22513c --- /dev/null +++ b/plays/jornalDeBrasilia.py @@ -0,0 +1,151 @@ +from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError + +from plays.base import BasePlay +from plays.items import EntryItem +from plog import logger + + +class JornalDeBrasiliaPlay(BasePlay): + name = "jornaldebrasilia" + + @classmethod + def match(cls, url: str) -> bool: + """Verifica se a URL pertence ao Jornal de Brasília.""" + return "jornaldebrasilia.com.br" in url + + def pre_run(self): + """Método executado antes do 'run'.""" + logger.info(f"[{self.name}] Preparando para extrair a URL: {self.url}") + + def run(self) -> EntryItem: + """Executa a extração principal dos dados da página.""" + with sync_playwright() as p: + browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080}) + page = browser.new_page( + ) + + page.route( + "**/*", + lambda route: route.abort() + if route.request.resource_type in {"image", "font", "media", "stylesheet"} + else route.continue_(), + ) + + page.set_default_navigation_timeout(60_000) + page.set_default_timeout(10_000) + + try: + page.goto(self.url, wait_until="domcontentloaded", timeout=60_000) + page.wait_for_timeout(2000) + except PlaywrightTimeoutError: + logger.error(f"[{self.name}] Timeout ao carregar a página: {self.url}") + browser.close() + return EntryItem(title="", url=self.url, description="", body="", tags=[]) + + + # Título + title = "" + title_selectors = [ + "h1.tdb-title-text", + ".wrap-title > h1", + "h1.title", + "h1.entry-title", + ".post-title", + ".content-head__title", + "h1" + ] + for selector in title_selectors: + try: + el = page.locator(selector).first + if el.is_visible(timeout=1000): + title = el.inner_text().strip() + if title: + logger.info(f"[{self.name}] Título encontrado com o seletor: '{selector}'") + break + except Exception: + continue + + if not title: + try: + title = page.title().strip().split("|")[0].strip() + except Exception: + title = "" + + + # Descrição (Subtítulo) + description = "" + description_selectors = [ + ".wrap-title > p:nth-child(2)", + "p.tdb-post-sub-title", + ".subtitle h2", + "p.td-post-sub-title" + ] + for selector in description_selectors: + try: + el = page.locator(selector).first + if el.is_visible(timeout=1000): + description = el.inner_text().strip() + if description: + logger.info(f"[{self.name}] Descrição encontrada com o seletor: '{selector}'") + break + except Exception: + continue + + # Corpo da Notícia + body = "" + body_container_selectors = [ + "div.tdb-block-inner.td-fix-index", + ".the-post-content", + ".td-post-content" + ] + for selector in body_container_selectors: + try: + container = page.locator(selector).first + if container.is_visible(timeout=1500): + logger.info(f"[{self.name}] Contêiner do corpo encontrado com o seletor: '{selector}'") + + # Extrai todos os parágrafos

de dentro do contêiner + paragraphs = container.locator("p") + parts = [] + for i in range(paragraphs.count()): + p_text = paragraphs.nth(i).inner_text().strip() + # Filtro para remover parágrafos indesejados + if p_text and len(p_text) > 40 and not p_text.lower().startswith(("leia mais", ">>", "crédito:", "siga o jornal")): + parts.append(p_text) + + if parts: + body = "\n\n".join(parts) + logger.info(f"[{self.name}] Corpo extraído com sucesso! Caracteres: {len(body)}") + break # Sai do loop principal se o corpo foi extraído + except Exception: + continue + + # Extração de Tags (opcional, pois nem sempre estão presentes) + tags = [] + tag_selectors = [ + ".tdb-tags a", + ".entities__list-item a", + ".entities__list li a" + ] + for selector in tag_selectors: + try: + tag_elements = page.locator(selector) + if tag_elements.count() > 0: + tag_list = [el.inner_text().strip() for el in tag_elements.all()] + tags = [t for t in tag_list if t] # Remove tags vazias + if tags: + logger.info(f"[{self.name}] {len(tags)} tags encontradas com o seletor: '{selector}'") + break + except Exception: + continue + + browser.close() + + return EntryItem( + title=title, + url=self.url, + description=description, + body=body, + tags=tags, + ) + diff --git a/spiders/__init__.py b/spiders/__init__.py index ecb5e1f..22a68e1 100644 --- a/spiders/__init__.py +++ b/spiders/__init__.py @@ -1,19 +1,20 @@ +from spiders.aliadosBrasil import AliadosBrasilSpider from spiders.base import BaseSpider from spiders.estadao import EstadaoSpider from spiders.folha import FolhaSpider +from spiders.gazetaDoPovo import GazetaDoPovoSpider from spiders.globo import GloboSpider from spiders.ig import IGSpider +from spiders.jornalDeBrasilia import JornalDeBrasiliaSpider +from spiders.maisGoias import MaisGoiasSpider from spiders.metropoles import MetropolesSpider from spiders.r7 import R7Spider from spiders.rbs import RBSSpider -from spiders.veja import VejaSpider from spiders.terra import TerraSpider from spiders.uol import UOLSpider -from spiders.gazetaDoPovo import GazetaDoPovoSpider -from spiders.maisGoias import MaisGoiasSpider -from spiders.aliadosBrasil import AliadosBrasilSpider +from spiders.veja import VejaSpider -__all__ = [ +_all_ = [ BaseSpider, EstadaoSpider, FolhaSpider, @@ -28,4 +29,5 @@ GazetaDoPovoSpider, MaisGoiasSpider, AliadosBrasilSpider, -] + JornalDeBrasiliaSpider, +] \ No newline at end of file diff --git a/spiders/jornalDeBrasilia.py b/spiders/jornalDeBrasilia.py new file mode 100644 index 0000000..7858e4c --- /dev/null +++ b/spiders/jornalDeBrasilia.py @@ -0,0 +1,91 @@ +import scrapy +from urllib.parse import urlparse +from spiders.base import BaseSpider +from spiders.items import URLItem + +class JornalDeBrasiliaSpider(BaseSpider): + name = "jornaldebrasiliaspider" + start_urls = ["https://jornaldebrasilia.com.br/"] + allowed_domains = ["jornaldebrasilia.com.br"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 2, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + } + } + + def allow_url(self, url: str) -> bool: + """ + Filtra apenas URLs de notícias válidas + """ + p = urlparse(url) + path = p.path.rstrip('/') + + # Blacklist de seções que não são notícias + blacklist = { + "/videos", "/podcasts", "/newsletter", "/contato", + "/sobre", "/anuncie", "/assine", "/busca" + } + + if path in blacklist: + return False + + # Requer pelo menos 2 segmentos (seção + slug) + segments = [seg for seg in path.split('/') if seg] + if len(segments) < 2: + return False + + # Verificar se é uma notícia (slug longo com hífens) + slug = segments[-1] + if slug.count('-') >= 2 or len(slug) > 20: + return True + + return False + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + ) + + def parse(self, response): + """ + Extrai URLs de notícias da página inicial + """ + selectors = [ + "article a[href]", + ".news-item a[href]", + ".headline a[href]", + "h2 a[href], h3 a[href]", + ] + + seen_urls = set() + + for selector in selectors: + links = response.css(selector) + + for link in links: + url = link.attrib.get("href") + if not url: + continue + + # Construir URL absoluta + full_url = response.urljoin(url) + full_url = full_url.split('#', 1)[0].split('?', 1)[0] + + # Evitar duplicatas + if full_url in seen_urls: + continue + seen_urls.add(full_url) + + # Filtrar URLs válidos + if self.allow_url(full_url): + yield URLItem(url=full_url)