diff --git a/Dockerfile b/Dockerfile index 1871764..97d34ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN /root/.local/bin/pipenv sync --dev # Instala minio e browsers no venv RUN /root/.local/bin/pipenv install minio==7.2.15 RUN /root/.local/bin/pipenv install scrapy-playwright==0.0.43 +ENV PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers RUN /root/.local/bin/pipenv run playwright install --with-deps firefox # Confirma que playwright foi instalado corretamente @@ -34,7 +35,8 @@ RUN apt-get update \ ENV PATH=/usr/src/.venv/bin:$PATH \ PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers WORKDIR /project diff --git a/Makefile b/Makefile index 12663c4..b8ba2e4 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,10 @@ scrape_uol: scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br + +scrape_jornaldaparaiba: + docker compose exec scraper python scrape_no_openai.py --platform jornaldaparaiba.com.br + # Crawler para todos os portais ou específicos crawl: @@ -112,6 +116,9 @@ crawl_ig: crawl_folha: docker compose run scraper python crawl.py folhaspider +crawl_jornaldaparaiba: + docker compose run scraper python crawl.py jornaldaparaiba + # Workflow completo de coleta de URLs crawl_all_working: @echo "Executando crawl de todos os portais funcionais..." @@ -123,6 +130,7 @@ crawl_all_working: @make crawl_aliadosBrasil @make crawl_ig @make crawl_folha + @make crawl_jornaldaparaiba @echo "Crawl de todos os portais concluído!" # Workflow completo de scraping @@ -136,6 +144,7 @@ scrape_all_working: @make scrape_r7 @make scrape_uol @make scrape_folha + @make scrape_jornaldaparaiba @echo "Scraping de todos os portais concluído!" # Pipeline completo: crawl + scrape @@ -219,25 +228,24 @@ run-frontend: # Rodar backend (server-web) com Docker run-backend-docker: - docker-compose up server-web + docker compose up server-web # Rodar frontend (client-web) com Docker run-frontend-docker: - docker-compose up client-web + docker compose up client-web # Construir e rodar backend (server-web) com Docker build-backend-docker: - docker-compose up --build server-web + docker compose up --build server-web # Construir e rodar frontend (client-web) com Docker build-frontend-docker: - docker-compose up --build client-web + docker compose up --build client-web # Rodar todos os serviços (backend, frontend, minio, db, scraper) all: - docker-compose up -d + docker compose up -d # Parar todos os serviços down: - docker-compose down - + docker compose down \ No newline at end of file diff --git a/create_db.py b/create_db.py index 617b85b..7ce5701 100644 --- a/create_db.py +++ b/create_db.py @@ -42,6 +42,7 @@ ("Gazeta do Povo", "https://www.gazetadopovo.com.br/", "gazetadopovo"), ("Mais Goiás", "https://www.maisgoias.com.br/", "maisgoias"), ("Aliados Brasil", "https://www.aliadosbrasiloficial.com.br/", "aliadosbrasil"), + ("Jornal da Paraíba", "https://jornaldaparaiba.com.br/", "jornaldaparaiba"), ] portals_to_add = [] for portal in portals: diff --git a/plays/__init__.py b/plays/__init__.py index 338fe28..8189816 100644 --- a/plays/__init__.py +++ b/plays/__init__.py @@ -11,6 +11,7 @@ from plays.gazetaDoPovo import GazetaDoPovoPlay from plays.maisGoias import MaisGoias from plays.aliadosBrasil import AliadosBrasilPlay +from plays.jornaldaparaiba import JornalDaParaibaPlay __all__ = [ ClicRBSPlay, @@ -26,4 +27,5 @@ GazetaDoPovoPlay, MaisGoias, AliadosBrasilPlay, + JornalDaParaibaPlay, ] diff --git a/plays/base.py b/plays/base.py index 7d81ff1..9d7c922 100644 --- a/plays/base.py +++ b/plays/base.py @@ -58,7 +58,7 @@ def get_scraper(cls, url, *args, **kwargs): def get_session_dir(self): if self.session_dir is None: - return f"./sessions/{self.name}_session/" + return f"./sessions/{self.name}_session/" return self.session_dir diff --git a/plays/jornaldaparaiba.py b/plays/jornaldaparaiba.py new file mode 100644 index 0000000..174de70 --- /dev/null +++ b/plays/jornaldaparaiba.py @@ -0,0 +1,141 @@ +import time +from playwright.sync_api import sync_playwright +from plays.base import BasePlay +from plays.items import EntryItem +from plog import logger + +class JornalDaParaibaPlay(BasePlay): + name = "jornaldaparaiba" + + @classmethod + def match(cls, url): + """Identifica se a URL é do Jornal da Paraíba""" + return "jornaldaparaiba.com.br" in url + + def pre_run(self): + """Configurações antes da execução""" + pass + + def run(self) -> EntryItem: + """Extrai conteúdo da notícia""" + with sync_playwright() as p: + browser = self.launch_browser(p, viewport={"width": 1920, "height": 1080}) + page = browser.new_page() + + logger.info(f"[{self.name}] Opening URL '{self.url}'...") + page.goto(self.url, timeout=180_000) + + # Aguardar carregamento + page.wait_for_selector("h1.jpb-article-title", timeout=30000) + + # 1. EXTRAIR TÍTULO + title = self._extract_title(page) + + # 2. EXTRAIR DESCRIÇÃO + description = self._extract_description(page) + + # 3. EXTRAIR CORPO + body = self._extract_body(page) + + # 4. EXTRAIR TAGS + tags = self._extract_tags(page) + + return EntryItem( + title=title, + url=self.url, + description=description, + body=body, + tags=tags, + ) + + def _extract_title(self, page): + """Extrai o título da notícia""" + selectors = [ + "h1.jpb-article-title", + ] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + title = element.first.inner_text().strip() + if title: + logger.info(f"[{self.name}] Title extracted: {title[:50]}...") + return title + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract title with {selector}: {str(e)}") + continue + + logger.error(f"[{self.name}] Failed to extract title") + return "" + + def _extract_description(self, page): + """Extrai a descrição/subtítulo da notícia""" + selectors = [ + "h2.jpb-article-subtitle", + ] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + description = element.first.inner_text().strip() + if description and len(description) > 10: + logger.info(f"[{self.name}] Description extracted: {description[:50]}...") + return description + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract description with {selector}: {str(e)}") + continue + + logger.info(f"[{self.name}] No description found") + return "" + + def _extract_body(self, page): + """Extrai o corpo da notícia""" + selectors = [ + "div#article", + ] + + for selector in selectors: + try: + element = page.locator(selector) + if element.count() > 0: + # Remover elementos indesejados + element.locator("script, style, .ad, .advertisement").evaluate_all("els => els.forEach(el => el.remove())") + + body = element.inner_text().strip() + if body and len(body) > 100: # Garantir que tem conteúdo substancial + logger.info(f"[{self.name}] Body extracted ({len(body)} chars)") + return body + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract body with {selector}: {str(e)}") + continue + + logger.error(f"[{self.name}] Failed to extract body") + return "" + + def _extract_tags(self, page): + """Extrai as tags da notícia""" + selectors = [ + "section.jpb-tags a", + ] + + tags = [] + + for selector in selectors: + try: + elements = page.locator(selector) + if elements.count() > 0: + for i in range(elements.count()): + tag_text = elements.nth(i).inner_text().strip() + if tag_text and tag_text not in tags: + tags.append(tag_text) + + if tags: + logger.info(f"[{self.name}] Tags extracted: {tags}") + break + except Exception as e: + logger.warning(f"[{self.name}] Failed to extract tags with {selector}: {str(e)}") + continue + + return tags diff --git a/spiders/__init__.py b/spiders/__init__.py index ecb5e1f..d2bb555 100644 --- a/spiders/__init__.py +++ b/spiders/__init__.py @@ -12,6 +12,7 @@ from spiders.gazetaDoPovo import GazetaDoPovoSpider from spiders.maisGoias import MaisGoiasSpider from spiders.aliadosBrasil import AliadosBrasilSpider +from spiders.jornaldaparaiba import JornalDaParaibaSpider __all__ = [ BaseSpider, @@ -28,4 +29,5 @@ GazetaDoPovoSpider, MaisGoiasSpider, AliadosBrasilSpider, + JornalDaParaibaSpider, ] diff --git a/spiders/jornaldaparaiba.py b/spiders/jornaldaparaiba.py new file mode 100644 index 0000000..f735f9b --- /dev/null +++ b/spiders/jornaldaparaiba.py @@ -0,0 +1,93 @@ +import scrapy +from urllib.parse import urlparse +from spiders.base import BaseSpider +from spiders.items import URLItem + +class JornalDaParaibaSpider(BaseSpider): + name = "jornaldaparaiba" + start_urls = ["https://jornaldaparaiba.com.br/"] + allowed_domains = ["jornaldaparaiba.com.br"] + + custom_settings = { + **BaseSpider.custom_settings, + "COOKIES_ENABLED": True, + "DOWNLOAD_DELAY": 2, + "DEFAULT_REQUEST_HEADERS": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + } + } + + def allow_url(self, url: str) -> bool: + """ + Filtra apenas URLs de notícias válidas + """ + p = urlparse(url) + path = p.path.rstrip('/') + + # Blacklist de seções que não são notícias + blacklist = { + "/videos", "/podcasts", "/newsletter", "/contato", + "/sobre", "/anuncie", "/assine", "/busca", "/expediente", + "/jornal-da-paraiba-25-anos", "/colunistas", "/guia-qualaboa", + "/vamos-trabalhar", "/educacao", "/espaco-opiniao" + } + + if any(blacklisted in path for blacklisted in blacklist): + return False + + # Requer pelo menos 2 segmentos (seção + slug) + segments = [seg for seg in path.split('/') if seg] + if len(segments) < 2: + return False + + # Verificar se é uma notícia (slug longo com hífens) + slug = segments[-1] + if slug.count('-') >= 2 or len(slug) > 20: + return True + + return False + + def start_requests(self): + for url in self.start_urls: + yield scrapy.Request( + url, + callback=self.parse, + dont_filter=True, + meta={"dont_redirect": True, "handle_httpstatus_list": [403]}, + ) + + def parse(self, response): + """ + Extrai URLs de notícias da página inicial + """ + selectors = [ + "section.jpb-chamadaBoxHorizontal a[href]", + "section.jpb-maisLidas a[href]", + "section.jpb-chamadaBox a[href]", + "section.jpb-ultimasNoticias a[href]", + ] + + seen_urls = set() + + for selector in selectors: + links = response.css(selector) + + for link in links: + url = link.attrib.get("href") + if not url: + continue + + # Construir URL absoluta + full_url = response.urljoin(url) + full_url = full_url.split('#', 1)[0].split('?', 1)[0] + + # Evitar duplicatas + if full_url in seen_urls: + continue + seen_urls.add(full_url) + + # Filtrar URLs válidos + if self.allow_url(full_url): + yield URLItem(url=full_url)