From 248ed30ac05c0c59f37121fb660edb2d1d842719 Mon Sep 17 00:00:00 2001 From: Carlos Nascimento Date: Wed, 8 Oct 2025 09:34:07 -0300 Subject: [PATCH 1/3] =?UTF-8?q?Adiciona=20corre=C3=A7=C3=A3o=20que=20insta?= =?UTF-8?q?la=20o=20firefox=20ao=20iniciar=20qualquer=20scraper.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 48 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index b8ba2e4..94fd123 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: start scrape scrape_no_openai crawl crawl_metropoles init_db migrate_db setup bash env stop wait-for-db prune +.PHONY: start scrape scrape_no_openai crawl crawl_metropoles init_db migrate_db setup bash env stop wait-for-db prune install_playwright test_playwright install_playwright_force # Configuração do ambiente env: @@ -11,7 +11,7 @@ env: fi # Configuração completa do projeto -setup: env start wait-for-db init_db migrate_db +setup: env start wait-for-db init_db migrate_db install_playwright @echo "Ambiente configurado com sucesso!" # Acesso ao shell do container @@ -52,35 +52,40 @@ wait-for-db: fi # Scraping com e sem OpenAI -scrape: +scrape: install_playwright docker compose run --rm scraper python scrape.py -scrape_no_openai: +scrape_no_openai: install_playwright docker compose run --rm scraper python scrape_no_openai.py # Scraping por plataforma específica (sem OpenAI) -scrape_metropoles: +scrape_metropoles: install_playwright docker compose exec scraper python scrape_no_openai.py --platform metropoles.com -scrape_maisgoias: +# Comando para verificação de instalação do Playwright +test_playwright: install_playwright + @echo "Verificando instalação do Playwright..." + docker compose exec scraper python -c "from playwright.sync_api import sync_playwright; print('Iniciando teste do Playwright'); p = sync_playwright().start(); print('Playwright iniciado com sucesso'); browser = p.firefox.launch(); print('Navegador lançado com sucesso'); page = browser.new_page(); print('Nova página criada'); page.goto('https://www.example.com'); print('Página carregada'); browser.close(); p.stop(); print('Teste do Playwright concluído com sucesso')" + +scrape_maisgoias: install_playwright docker compose exec scraper python scrape_no_openai.py --platform maisgoias.com.br -scrape_aliadosbrasil: +scrape_aliadosbrasil: install_playwright docker compose exec scraper python scrape_no_openai.py --platform aliadosbrasiloficial.com.br -scrape_ig: +scrape_ig: install_playwright docker compose exec scraper python scrape_no_openai.py --platform ig.com.br -scrape_veja: +scrape_veja: install_playwright docker compose exec scraper python scrape_no_openai.py --platform veja.abril.com.br -scrape_r7: +scrape_r7: install_playwright docker compose exec scraper python scrape_no_openai.py --platform r7.com -scrape_uol: +scrape_uol: install_playwright docker compose exec scraper python scrape_no_openai.py --platform uol.com.br -scrape_folha: +scrape_folha: install_playwright docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br scrape_jornaldaparaiba: @@ -170,6 +175,22 @@ collect: crawl_metropoles scrape_no_openai collect_working: crawl_all_working scrape_all_working @echo "Coleta completa de todos os portais funcionais concluída!" +# Instala navegadores do Playwright com as permissões corretas +install_playwright: + @echo "Instalando navegadores do Playwright com as permissões adequadas..." + docker compose exec --user root scraper bash -c "mkdir -p /project/.cache && chmod 777 /project/.cache && source /usr/src/.venv/bin/activate && PLAYWRIGHT_BROWSERS_PATH=/project/.cache/ms-playwright playwright install --with-deps firefox" + docker compose exec --user root scraper bash -c "mkdir -p /project/.cache/ms-playwright && chmod -R 777 /project/.cache/ms-playwright" + @echo "Navegador Firefox instalado com sucesso para o Playwright!" + +# Instala navegadores do Playwright de forma forçada (para casos de erro persistente) +install_playwright_force: + @echo "Forçando reinstalação completa dos navegadores do Playwright..." + docker compose exec --user root scraper bash -c "rm -rf /project/.cache/ms-playwright || true" + docker compose exec --user root scraper bash -c "mkdir -p /project/.cache/ms-playwright && chmod 777 /project/.cache/ms-playwright" + docker compose exec --user root scraper bash -c "source /usr/src/.venv/bin/activate && PLAYWRIGHT_BROWSERS_PATH=/project/.cache/ms-playwright playwright install --with-deps firefox" + docker compose exec --user root scraper bash -c "chmod -R 777 /project/.cache/ms-playwright" + @echo "Navegador Firefox reinstalado com sucesso para o Playwright!" + # Exibir ajuda help: @echo "Check-up - Ferramenta de análise de anúncios de saúde" @@ -211,6 +232,9 @@ help: @echo " make init_db - Inicializa as tabelas do banco de dados" @echo " make migrate_db - Executa as migrações do banco de dados" @echo " make prune - Remove containers, redes e imagens não utilizadas" + @echo " make install_playwright - Instala navegadores do Playwright com as permissões corretas" + @echo " make install_playwright_force - Reinstala navegadores do Playwright de forma forçada" + @echo " make test_playwright - Testa se a instalação do Playwright está funcionando" .PHONY: start scrape scrape_no_openai scrape_fixed scrape_working install run-backend run-frontend docker-run-backend docker-run-frontend docker-build-backend docker-build-frontend all stop down From 73ad70ebebb63c0a9316f4a60f8ef1b8bb091578 Mon Sep 17 00:00:00 2001 From: Carlos Nascimento Date: Wed, 8 Oct 2025 13:57:51 -0300 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20Ajustando=20instala=C3=A7=C3=A3o=20d?= =?UTF-8?q?o=20firefox=20no=20playwrigth?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: alanagabriele --- Dockerfile | 40 +++++++++++++++++++++++----------------- Makefile | 43 +++++++++++++------------------------------ 2 files changed, 36 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index 97d34ca..b23f678 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,20 @@ -# builder +# ========== builder ========== FROM python:3.12.3 AS builder RUN pip install --user pipenv==2023.12.1 ENV PIPENV_VENV_IN_PROJECT=1 +# onde os binários dos browsers ficarão (fora do home) +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright COPY Pipfile Pipfile.lock /usr/src/ WORKDIR /usr/src -# Instala dependências com pipenv RUN /root/.local/bin/pipenv sync --dev - -# Instala minio e browsers no venv RUN /root/.local/bin/pipenv install minio==7.2.15 RUN /root/.local/bin/pipenv install scrapy-playwright==0.0.43 -ENV PLAYWRIGHT_BROWSERS_PATH=/usr/src/.venv/pw-browsers -RUN /root/.local/bin/pipenv run playwright install --with-deps firefox + +# baixa SOMENTE os browsers (sem --with-deps aqui) +RUN /root/.local/bin/pipenv run playwright install firefox # Confirma que playwright foi instalado corretamente RUN /usr/src/.venv/bin/python -c "import playwright; print(playwright)" @@ -22,16 +22,23 @@ RUN /usr/src/.venv/bin/python -c "import playwright; print(playwright)" # runtime FROM python:3.12.3-slim AS runtime -COPY --from=builder /usr/src/.venv/ /usr/src/.venv/ +# mesmo path de browsers no runtime +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +# dependências do Firefox +RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 \ - libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 \ - libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libstdc++6 libx11-6 \ - libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 \ - libxi6 libxrandr2 libxrender1 libxss1 libxtst6 wget xdg-utils \ - && rm -rf /var/lib/apt/lists/* + libc6 libcairo2 libcups2 libdbus-1-3 libdbus-glib-1-2 libexpat1 libfontconfig1 \ + libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 \ + libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 \ + libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxshmfence1 \ + libxss1 libxtst6 wget xdg-utils \ + && rm -rf /var/lib/apt/lists/* + +# venv do app +COPY --from=builder /usr/src/.venv/ /usr/src/.venv/ +# binários dos browsers (copiados do builder) +COPY --from=builder /ms-playwright/ /ms-playwright/ ENV PATH=/usr/src/.venv/bin:$PATH \ PYTHONDONTWRITEBYTECODE=1 \ @@ -42,7 +49,7 @@ WORKDIR /project RUN groupadd -r scraper && \ useradd --no-log-init -r -g scraper -d /project scraper && \ - chown -R scraper:scraper /project /usr/src/.venv + chown -R scraper:scraper /project /usr/src/.venv /ms-playwright COPY --chown=scraper:scraper . /project @@ -51,5 +58,4 @@ USER scraper HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD python -c "import sys; sys.exit(0)" - -CMD ["ipython"] +CMD ["ipython"] \ No newline at end of file diff --git a/Makefile b/Makefile index 94fd123..b7ae1b2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: start scrape scrape_no_openai crawl crawl_metropoles init_db migrate_db setup bash env stop wait-for-db prune install_playwright test_playwright install_playwright_force +.PHONY: start scrape scrape_no_openai crawl crawl_metropoles init_db migrate_db setup bash env stop wait-for-db prune test_playwright # Configuração do ambiente env: @@ -11,7 +11,7 @@ env: fi # Configuração completa do projeto -setup: env start wait-for-db init_db migrate_db install_playwright +setup: env start wait-for-db init_db migrate_db @echo "Ambiente configurado com sucesso!" # Acesso ao shell do container @@ -52,40 +52,40 @@ wait-for-db: fi # Scraping com e sem OpenAI -scrape: install_playwright +scrape: docker compose run --rm scraper python scrape.py -scrape_no_openai: install_playwright +scrape_no_openai: docker compose run --rm scraper python scrape_no_openai.py # Scraping por plataforma específica (sem OpenAI) -scrape_metropoles: install_playwright +scrape_metropoles: docker compose exec scraper python scrape_no_openai.py --platform metropoles.com # Comando para verificação de instalação do Playwright -test_playwright: install_playwright +test_playwright: @echo "Verificando instalação do Playwright..." docker compose exec scraper python -c "from playwright.sync_api import sync_playwright; print('Iniciando teste do Playwright'); p = sync_playwright().start(); print('Playwright iniciado com sucesso'); browser = p.firefox.launch(); print('Navegador lançado com sucesso'); page = browser.new_page(); print('Nova página criada'); page.goto('https://www.example.com'); print('Página carregada'); browser.close(); p.stop(); print('Teste do Playwright concluído com sucesso')" -scrape_maisgoias: install_playwright +scrape_maisgoias: docker compose exec scraper python scrape_no_openai.py --platform maisgoias.com.br -scrape_aliadosbrasil: install_playwright +scrape_aliadosbrasil: docker compose exec scraper python scrape_no_openai.py --platform aliadosbrasiloficial.com.br -scrape_ig: install_playwright +scrape_ig: docker compose exec scraper python scrape_no_openai.py --platform ig.com.br -scrape_veja: install_playwright +scrape_veja: docker compose exec scraper python scrape_no_openai.py --platform veja.abril.com.br -scrape_r7: install_playwright +scrape_r7: docker compose exec scraper python scrape_no_openai.py --platform r7.com -scrape_uol: install_playwright +scrape_uol: docker compose exec scraper python scrape_no_openai.py --platform uol.com.br -scrape_folha: install_playwright +scrape_folha: docker compose exec scraper python scrape_no_openai.py --platform folha.uol.com.br scrape_jornaldaparaiba: @@ -175,21 +175,6 @@ collect: crawl_metropoles scrape_no_openai collect_working: crawl_all_working scrape_all_working @echo "Coleta completa de todos os portais funcionais concluída!" -# Instala navegadores do Playwright com as permissões corretas -install_playwright: - @echo "Instalando navegadores do Playwright com as permissões adequadas..." - docker compose exec --user root scraper bash -c "mkdir -p /project/.cache && chmod 777 /project/.cache && source /usr/src/.venv/bin/activate && PLAYWRIGHT_BROWSERS_PATH=/project/.cache/ms-playwright playwright install --with-deps firefox" - docker compose exec --user root scraper bash -c "mkdir -p /project/.cache/ms-playwright && chmod -R 777 /project/.cache/ms-playwright" - @echo "Navegador Firefox instalado com sucesso para o Playwright!" - -# Instala navegadores do Playwright de forma forçada (para casos de erro persistente) -install_playwright_force: - @echo "Forçando reinstalação completa dos navegadores do Playwright..." - docker compose exec --user root scraper bash -c "rm -rf /project/.cache/ms-playwright || true" - docker compose exec --user root scraper bash -c "mkdir -p /project/.cache/ms-playwright && chmod 777 /project/.cache/ms-playwright" - docker compose exec --user root scraper bash -c "source /usr/src/.venv/bin/activate && PLAYWRIGHT_BROWSERS_PATH=/project/.cache/ms-playwright playwright install --with-deps firefox" - docker compose exec --user root scraper bash -c "chmod -R 777 /project/.cache/ms-playwright" - @echo "Navegador Firefox reinstalado com sucesso para o Playwright!" # Exibir ajuda help: @@ -232,8 +217,6 @@ help: @echo " make init_db - Inicializa as tabelas do banco de dados" @echo " make migrate_db - Executa as migrações do banco de dados" @echo " make prune - Remove containers, redes e imagens não utilizadas" - @echo " make install_playwright - Instala navegadores do Playwright com as permissões corretas" - @echo " make install_playwright_force - Reinstala navegadores do Playwright de forma forçada" @echo " make test_playwright - Testa se a instalação do Playwright está funcionando" .PHONY: start scrape scrape_no_openai scrape_fixed scrape_working install run-backend run-frontend docker-run-backend docker-run-frontend docker-build-backend docker-build-frontend all stop down From d18f05ae380d5503321d0c588ce2a0e610b9c127 Mon Sep 17 00:00:00 2001 From: Luciano Freitas Date: Tue, 28 Oct 2025 10:23:45 -0300 Subject: [PATCH 3/3] build(Makefile): melhoria na execucao dos containers scraper --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b7ae1b2..a7f7ce2 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ scrape_metropoles: # Comando para verificação de instalação do Playwright test_playwright: @echo "Verificando instalação do Playwright..." - docker compose exec scraper python -c "from playwright.sync_api import sync_playwright; print('Iniciando teste do Playwright'); p = sync_playwright().start(); print('Playwright iniciado com sucesso'); browser = p.firefox.launch(); print('Navegador lançado com sucesso'); page = browser.new_page(); print('Nova página criada'); page.goto('https://www.example.com'); print('Página carregada'); browser.close(); p.stop(); print('Teste do Playwright concluído com sucesso')" + docker compose run --rm scraper python -c "from playwright.sync_api import sync_playwright; print('Iniciando teste do Playwright'); p = sync_playwright().start(); print('Playwright iniciado com sucesso'); browser = p.firefox.launch(); print('Navegador lançado com sucesso'); page = browser.new_page(); print('Nova página criada'); page.goto('https://www.example.com'); print('Página carregada'); browser.close(); p.stop(); print('Teste do Playwright concluído com sucesso')" scrape_maisgoias: docker compose exec scraper python scrape_no_openai.py --platform maisgoias.com.br