From c58f936e9a842100e7302ffe09481cbdd3a96404 Mon Sep 17 00:00:00 2001 From: Andrej Luptak Date: Wed, 2 Sep 2026 13:29:43 +0200 Subject: [PATCH 1/3] chore: replace git dependency for http Replaced git dependency for playboooks and content import with http. This includes change from git clone to downloading an archive. Updated requirements, keeping git for dev only as tests requires them. RHINENG-30176 and RHINENG-30177 --- Dockerfile | 33 ++++-- common/config.py | 4 +- conf/taskomatic.env | 4 +- deploy/clowdapp.yaml | 17 +-- poetry.lock | 8 +- pyproject.toml | 2 +- requirements-build.txt | 1 - requirements-dev.txt | 9 ++ requirements.txt | 9 -- taskomatic/jobs/archive_downloader.py | 106 ++++++++++++++++++ taskomatic/jobs/rules_git_sync.py | 96 ++++++++++------ tests/taskomatic_tests/test_rules_git_sync.py | 83 ++++++++++++++ 12 files changed, 300 insertions(+), 72 deletions(-) create mode 100644 taskomatic/jobs/archive_downloader.py diff --git a/Dockerfile b/Dockerfile index 491918415..10d36d299 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ ARG ALT_REPO # Install PostgreSQL without pg_repack (will copy from builder) RUN (microdnf module enable -y postgresql:18 || curl -o /etc/yum.repos.d/postgresql.repo $ALT_REPO) && \ microdnf install -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \ - python312 libpq shadow-utils git-core postgresql && \ + python312 libpq shadow-utils postgresql && \ microdnf clean all # Copy pg_repack 1.5.2 from builder stage @@ -63,18 +63,33 @@ ENV MINIMAL_SCHEMA=172 # Baked-in content for FedRAMP ARG STATIC_ASSETS=0 +ARG CONTENT_ARCHIVE_URL="https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz" +ARG PLAYBOOKS_ARCHIVE_URL="https://gitlab.cee.redhat.com/insights-rules/insights-playbooks/-/archive/master/insights-playbooks-master.tar.gz" +ARG VMAAS_ARCHIVE_URL="https://gitlab.cee.redhat.com/vmaas/vmaas-assets/-/archive/master/vmaas-assets-master.tar.gz" +ARG ARCHIVE_EXT=".tar.gz" ARG GIT_TOKEN="" + RUN if [ "${STATIC_ASSETS}" == 1 ] ; then \ curl -o /etc/pki/ca-trust/source/anchors/2022-IT-Root-CA.crt https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem && \ update-ca-trust extract && \ - git clone --depth 1 https://gitlab.cee.redhat.com/vmaas/vmaas-assets.git /engine/vmaas_assets_git && \ - git clone --depth 1 https://gitlab.cee.redhat.com/insights-rules/insights-playbooks.git /engine/insights_playbooks_git && \ - git clone --depth 1 "https://$GIT_TOKEN@github.com/RedHatInsights/insights-content-vulnerability.git" /engine/insights_content_vulnerability_git && \ - # below is needed to avoid git 'detected dubious ownership' error when running as a rootless container... - git config --system --add safe.directory /engine/vmaas_assets_git && \ - git config --system --add safe.directory /engine/insights_playbooks_git && \ - git config --system --add safe.directory /engine/insights_content_vulnerability_git && \ - echo "Cloned static assets" ; \ + \ + # VMaaS assets + curl -L "${VMAAS_ARCHIVE_URL}" -o /tmp/vmaas${ARCHIVE_EXT} && \ + tar -xzf /tmp/vmaas${ARCHIVE_EXT} -C /tmp && \ + mv /tmp/vmaas-assets-* /engine/vmaas_assets_git && \ + rm /tmp/vmaas${ARCHIVE_EXT} && \ + \ + # Insights playbooks, we need to keep the archive for sha computation + curl -L "${PLAYBOOKS_ARCHIVE_URL}" -o /engine/insights_playbooks_git${ARCHIVE_EXT} && \ + tar -xzf /engine/insights_playbooks_git${ARCHIVE_EXT} -C /tmp && \ + mv /tmp/insights-playbooks-* /engine/insights_playbooks_git && \ + \ + # Insights content, we need to keep the archive for sha computation + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${CONTENT_ARCHIVE_URL}" -o /engine/insights_content_vulnerability_git${ARCHIVE_EXT} && \ + tar -xzf /engine/insights_content_vulnerability_git${ARCHIVE_EXT} -C /tmp && \ + mv /tmp/insights-content-vulnerability-* /engine/insights_content_vulnerability_git && \ + \ + echo "Downloaded static assets via HTTP" ; \ fi USER insights diff --git a/common/config.py b/common/config.py index 0b2901858..cfe9131de 100644 --- a/common/config.py +++ b/common/config.py @@ -101,8 +101,8 @@ def __init__(self): # taskomatic jobs self.system_deletion_threshold = int(os.getenv("SYSTEM_DELETION_THRESHOLD", "24")) # 24 hours - self.content_git_repo = os.getenv("CONTENT_GIT_REPO", "") - self.playbooks_git_repo = os.getenv("PLAYBOOKS_GIT_REPO", "") + self.content_archive_url = os.getenv("CONTENT_ARCHIVE_URL", "") + self.playbooks_archive_url = os.getenv("PLAYBOOKS_ARCHIVE_URL", "") self.git_token = os.getenv("GIT_TOKEN", "") # used also by exploit-sync self.jobs = os.getenv("JOBS", "").split(",") self.jobs_startup = os.getenv("JOBS_STARTUP", "").split(",") diff --git a/conf/taskomatic.env b/conf/taskomatic.env index 4dd3875e6..23d366d58 100644 --- a/conf/taskomatic.env +++ b/conf/taskomatic.env @@ -1,8 +1,8 @@ JOBS=stale_systems:5,delete_systems:30,rules_git_sync:240,db_metrics:15,cacheman:5,delete_notifications:720,migrate_data:999999 JOBS_STARTUP=db_metrics,cacheman,migrate_data SYSTEM_DELETION_THRESHOLD=1 -CONTENT_GIT_REPO= -PLAYBOOKS_GIT_REPO= +CONTENT_ARCHIVE_URL= +PLAYBOOKS_ARCHIVE_URL= ACCOUNTS_BLACKLIST= CACHE_MINIMAL_ACCOUNT_SYSTEMS=400 CACHE_KEEP_KEEPALIVE_HOURS=168 diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 1ef52994e..e27997882 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -225,10 +225,10 @@ objects: value: ${JOBS} - name: JOBS_STARTUP value: ${JOBS_STARTUP} - - name: CONTENT_GIT_REPO - value: ${CONTENT_GIT} - - name: PLAYBOOKS_GIT_REPO - value: https://github.com/RedHatInsights/insights-playbooks.git + - name: CONTENT_ARCHIVE_URL + value: ${CONTENT_ARCHIVE} + - name: PLAYBOOKS_ARCHIVE_URL + value: ${PLAYBOOKS_ARCHIVE} - name: GIT_TOKEN valueFrom: secretKeyRef: @@ -1202,9 +1202,12 @@ parameters: - name: JOBS_STARTUP description: Jobs to run on app startup (restart) value: "db_metrics,cacheman,migrate_data" -- name: CONTENT_GIT - description: Location of insights-content(-vulnerability) repo - value: https://github.com/RedHatInsights/insights-content-vulnerability.git +- name: CONTENT_ARCHIVE + description: Archive URL for insights-content-vulnerability archive download + value: https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz +- name: PLAYBOOKS_ARCHIVE + description: Archive URL for insights-playbooks archive download + value: https://gitlab.cee.redhat.com/insights-rules/insights-playbooks/-/archive/master/insights-playbooks-master.tar.gz - name: SYSTEM_DELETION_THRESHOLD description: Delete systems marked as deleted before N hours value: "1" diff --git a/poetry.lock b/poetry.lock index 8bab44639..c76ce2e0e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1499,7 +1499,7 @@ version = "4.0.12" description = "Git Object Database" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, @@ -1514,7 +1514,7 @@ version = "3.1.62" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "gitpython-3.1.62-py3-none-any.whl", hash = "sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e"}, {file = "gitpython-3.1.62.tar.gz", hash = "sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af"}, @@ -3824,7 +3824,7 @@ version = "5.0.3" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f"}, {file = "smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c"}, @@ -4323,4 +4323,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.14" -content-hash = "31d13d686be3946ce7cf476d620e2b70ec61101fd98b231a279f2e76660de5e0" +content-hash = "74255851a12804a03629ad0ca2c417280a2cc2b7b2a1a27ffbb99ed675fc6361" diff --git a/pyproject.toml b/pyproject.toml index a1007984a..d0607f985 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ botocore = "^1.34.122" # Pinned due to build issue in Konflux cramjam = "=2.10.0" flask = "^3.1.1" -gitpython = "^3.1.43" connexion = {version = "^3.1.0", extras = ["swagger-ui", "flask"]} gunicorn = "^23.0.0" peewee = "^3.18.1" @@ -76,6 +75,7 @@ python-box = "^7.0.0" insights-core = "^3.3.2" pytest-cov = "^7.0.0" pytest-aiohttp = "^1.0.5" +gitpython = "^3.1.43" # Tests use this [build-system] requires = ["poetry-core>=2.4.1", "setuptools-rust==1.11.1", "puccinialin==0.1"] diff --git a/requirements-build.txt b/requirements-build.txt index 8449ccbb3..632490139 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -531,7 +531,6 @@ setuptools==84.0.0 \ # charset-normalizer # cryptography # frozenlist - # gitpython # googleapis-common-protos # grpcio # gunicorn diff --git a/requirements-dev.txt b/requirements-dev.txt index 95cb060c5..52a4a9ecf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -580,6 +580,12 @@ frozenlist==1.8.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd +gitdb==4.0.12 ; python_version >= "3.12" and python_version < "3.14" \ + --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \ + --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf +gitpython==3.1.62 ; python_version >= "3.12" and python_version < "3.14" \ + --hash=sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af \ + --hash=sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e identify==2.6.19 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a \ --hash=sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842 @@ -1221,6 +1227,9 @@ scramp==1.4.17 ; python_version >= "3.12" and python_version < "3.14" \ six==1.17.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 +smmap==5.0.3 ; python_version >= "3.12" and python_version < "3.14" \ + --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \ + --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f testing-common-database==2.0.3 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:965d80b2985315325dc358c3061b174a712f4d4d5bf6a80b58b11f9a1dd86d73 \ --hash=sha256:e3ed492bf480a87f271f74c53b262caf5d85c8bc09989a8f534fa2283ec52492 diff --git a/requirements.txt b/requirements.txt index ba899fcc4..482989584 100644 --- a/requirements.txt +++ b/requirements.txt @@ -891,12 +891,6 @@ frozenlist==1.8.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd -gitdb==4.0.12 ; python_version >= "3.12" and python_version < "3.14" \ - --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \ - --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf -gitpython==3.1.62 ; python_version >= "3.12" and python_version < "3.14" \ - --hash=sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af \ - --hash=sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e google-auth==2.58.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd \ --hash=sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d @@ -1855,9 +1849,6 @@ semver==3.0.4 ; python_version >= "3.12" and python_version < "3.14" \ six==1.17.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 -smmap==5.0.3 ; python_version >= "3.12" and python_version < "3.14" \ - --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \ - --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f starlette==1.6.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \ --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b diff --git a/taskomatic/jobs/archive_downloader.py b/taskomatic/jobs/archive_downloader.py new file mode 100644 index 000000000..1795fca03 --- /dev/null +++ b/taskomatic/jobs/archive_downloader.py @@ -0,0 +1,106 @@ +"""Generic HTTP archive downloader.""" + +import hashlib +import os +import shutil +import tarfile +import zipfile +from pathlib import Path +from typing import Tuple +from urllib.parse import urlparse + +import requests + +from common.logging import get_logger + +LOGGER = get_logger(__name__) + +# Supported archive formats (order matters for detection) +SUPPORTED_ARCHIVE_FORMATS = {".tar.gz": "tarball", ".zip": "zip"} + + +def _get_archive_type(archive_path: str) -> str: + """Detect archive type from path. Returns 'tarball', 'zip', or None""" + for ext, archive_type in SUPPORTED_ARCHIVE_FORMATS.items(): + if archive_path.endswith(ext): + return archive_type + return None + + +def extract_commit_sha_from_archive(archive_path: str) -> str: + """ + Extract commit SHA from local archive file. + Returns commit SHA (from PAX headers/ZIP comment, or SHA256 hash as fallback) + """ + archive_type = _get_archive_type(archive_path) + + try: + if archive_type == "zip": + with zipfile.ZipFile(archive_path, "r") as z: + if sha := z.comment.decode("utf-8").strip(): + LOGGER.debug("Extracted commit SHA from ZIP comment: %s", sha[:8]) + return sha + elif archive_type == "tarball": + with tarfile.open(archive_path, "r:gz") as t: + if (first := next(iter(t), None)) and first.pax_headers: + if sha := first.pax_headers.get("comment"): + LOGGER.debug("Extracted commit SHA from PAX headers: %s", sha[:8]) + return sha + else: + LOGGER.error("Unsupported archive format %s", archive_path) + except Exception as err: + LOGGER.warning("Could not extract commit SHA: %s", err) + + # Fallback: SHA256 hash of archive + LOGGER.warning("No metadata SHA, using SHA256 hash of archive") + with open(archive_path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest()[:40] + + +class ArchiveDownloader: + """Generic downloader for tar.gz and zip archives""" + + def __init__(self, archive_url: str, auth_headers: dict = None): + self.archive_url = archive_url + self.auth_headers = auth_headers or {} + + def _extract_archive(self, archive_path: str, extract_to: str) -> None: + archive_type = _get_archive_type(archive_path) + + if archive_type == "zip": + with zipfile.ZipFile(archive_path, "r") as z: + z.extractall(extract_to) + elif archive_type == "tarball": + with tarfile.open(archive_path, "r:gz") as t: + t.extractall(extract_to, filter="data") + else: + raise ValueError(f"Unsupported archive format: {archive_path}") + + def download_and_extract(self, extract_to: str) -> Tuple[str, str]: + """Downloads, extracts, and returns (extracted_dir_path, commit_sha)""" + os.makedirs(extract_to, exist_ok=True) + # Detect extension from URL path (handles query params, supports multi-suffix like .tar.gz) + url_path = urlparse(self.archive_url).path + ext = "".join(Path(url_path).suffixes).lstrip(".") + archive_path = os.path.join(extract_to, f"archive.{ext}") + + LOGGER.info("Downloading %s", self.archive_url) + with requests.get(self.archive_url, headers=self.auth_headers, stream=True, timeout=300) as r: + r.raise_for_status() + with open(archive_path, "wb") as f: + # tell Python to stream data securely from the source to drive + shutil.copyfileobj(r.raw, f) + + commit_sha = extract_commit_sha_from_archive(archive_path) + + self._extract_archive(archive_path, extract_to) + os.remove(archive_path) + + # Unpacked archive may have a folder with a different naming inside, find the newly extracted folder + dirs = [os.path.join(extract_to, d) for d in os.listdir(extract_to) if os.path.isdir(os.path.join(extract_to, d))] + if not dirs: # We should always have top level directory, so no fallback here + raise RuntimeError(f"Archive extraction failed: no directories found in {extract_to}") + extracted_dir = dirs[0] + + LOGGER.debug("Extracted to: %s (SHA: %s)", extracted_dir, commit_sha[:8]) + return extracted_dir, commit_sha diff --git a/taskomatic/jobs/rules_git_sync.py b/taskomatic/jobs/rules_git_sync.py index 65cde4571..6aa72ba4f 100755 --- a/taskomatic/jobs/rules_git_sync.py +++ b/taskomatic/jobs/rules_git_sync.py @@ -4,19 +4,22 @@ """ import os +import shutil import tempfile from datetime import datetime from functools import reduce +from typing import Tuple import yaml -from git import Repo -from git.cmd import GitCommandError from psycopg2.extras import execute_values from common.config import Config from common.logging import get_logger from common.logging import init_logging from common.utils import EXCEPTION_COUNT +from taskomatic.jobs.archive_downloader import SUPPORTED_ARCHIVE_FORMATS +from taskomatic.jobs.archive_downloader import ArchiveDownloader +from taskomatic.jobs.archive_downloader import extract_commit_sha_from_archive from taskomatic.jobs.common import get_conn LOGGER = get_logger(__name__) @@ -29,7 +32,7 @@ def store_versions(conn, content_version, playbooks_version): """ - Store SHA of latest imported commit and URL of git repositories used + Store SHA of latest imported commit and URL of git archive repositories used Args: content_version str: sha of HEAD of insights-content-vulnerability @@ -44,9 +47,9 @@ def store_versions(conn, content_version, playbooks_version): insights_playbooks_repo = %(playbooks_repo)s""", { "content_version": content_version, - "content_repo": CFG.content_git_repo, + "content_repo": CFG.content_archive_url, "playbooks_version": playbooks_version, - "playbooks_repo": CFG.playbooks_git_repo, + "playbooks_repo": CFG.playbooks_archive_url, }, ) @@ -520,37 +523,22 @@ def sync(tmpdirname: str, content_version: str, playbooks_version: str): # pyli conn.close() -def clone_repo(git_repo, repo_dir_name): - """Clone single git repo""" - if not git_repo: - LOGGER.error("Git repo for %s not set.", repo_dir_name) - return False - git_repo = git_repo.replace("github.com", "%s@github.com" % CFG.git_token) if CFG.git_token else git_repo - try: - repo = Repo.clone_from(git_repo, repo_dir_name, depth=1) - except GitCommandError: - LOGGER.exception("Error cloning repo: ") - return False - return repo.head.object.hexsha - - -def clone_repos(tmpdirname): - """Clone git repos to temp dir""" - revisions = ( - clone_repo(CFG.content_git_repo, f"{tmpdirname}/{CONTENT_GIT_NAME}"), - clone_repo(CFG.playbooks_git_repo, f"{tmpdirname}/{PLAYBOOKS_GIT_NAME}"), - ) - return all(revisions), revisions +def load_static_repo(repo_dir_name): + """Get version of static repo (FedRAMP mode)""" + repo_name = os.path.basename(repo_dir_name) + # Try all supported archive formats + for ext in SUPPORTED_ARCHIVE_FORMATS.keys(): + archive_path = os.path.join(STATIC_GITS_PATH, f"{repo_name}{ext}") + if os.path.exists(archive_path): + try: + return extract_commit_sha_from_archive(archive_path) + except Exception: + LOGGER.exception("Error loading repo SHA from archive: %s", archive_path) + return False -def load_static_repo(repo_dir_name): - """Get revision of single git repo""" - try: - repo = Repo(repo_dir_name) - except GitCommandError: - LOGGER.exception("Error loading repo: ") - return False - return repo.head.object.hexsha + LOGGER.error("No archive found for %s in %s", repo_name, STATIC_GITS_PATH) + return False def load_static_repos(dirname): @@ -562,15 +550,49 @@ def load_static_repos(dirname): return all(revisions), revisions +def download_repo(archive_url: str, auth_headers: dict, target_dir: str) -> str | None: + """Download repository using HTTP archive""" + if not archive_url: + LOGGER.error("Archive URL not provided.") + return None + + try: + # Use tmp dir for download and extract + # (to prevent nested directories and file rewriting during concurent download) + with tempfile.TemporaryDirectory() as dl_tmp: + downloader = ArchiveDownloader(archive_url, auth_headers) + extracted_path, commit_sha = downloader.download_and_extract(dl_tmp) + shutil.move(extracted_path, target_dir) + + return commit_sha + except Exception as err: + LOGGER.exception("Error downloading repo: %s", err) + from common.utils import EXCEPTION_COUNT + + EXCEPTION_COUNT.inc() + return None + + +def download_repos(tmpdirname: str) -> Tuple[bool, Tuple[str, str]]: + """Download content and playbooks repos""" + auth_headers = {"Authorization": f"Bearer {CFG.git_token}"} if CFG.git_token.strip() else {} + + content_sha = download_repo(CFG.content_archive_url, auth_headers, f"{tmpdirname}/{CONTENT_GIT_NAME}") + playbooks_sha = download_repo(CFG.playbooks_archive_url, auth_headers, f"{tmpdirname}/{PLAYBOOKS_GIT_NAME}") + + revisions = (content_sha, playbooks_sha) + return all(revisions), revisions + + def run(): """Application entrypoint""" LOGGER.info("Started rules_git_sync job.") if not CFG.is_fedramp: with tempfile.TemporaryDirectory() as tmpdirname: - LOGGER.info("Cloning repos.") - success, revisions = clone_repos(tmpdirname) + LOGGER.info("Downloading repos via HTTP.") + success, revisions = download_repos(tmpdirname) if success: - LOGGER.info("Cloning repos successful, starting sync.") + LOGGER.info("Download successful, starting sync.") sync(tmpdirname, revisions[0], revisions[1]) else: LOGGER.info("FedRAMP env, using static repositories.") diff --git a/tests/taskomatic_tests/test_rules_git_sync.py b/tests/taskomatic_tests/test_rules_git_sync.py index 566b52606..26ae8b9fb 100644 --- a/tests/taskomatic_tests/test_rules_git_sync.py +++ b/tests/taskomatic_tests/test_rules_git_sync.py @@ -59,3 +59,86 @@ def test_sync(pg_db_conn, monkeypatch): # pylint: disable=unused-argument assert row[1] is None cur.close() + + @staticmethod + def test_download_and_extract(tmp_path, monkeypatch): + """Test ArchiveDownloader downloads, extracts and cleans up""" + import io + import tarfile + from unittest.mock import MagicMock + from unittest.mock import Mock + + from taskomatic.jobs.archive_downloader import ArchiveDownloader + + # Create tarball in memory + tar_bytes = io.BytesIO() + with tarfile.open(fileobj=tar_bytes, mode="w:gz", format=tarfile.PAX_FORMAT) as tar: + tarinfo = tarfile.TarInfo(name="test-repo-main/README.md") + tarinfo.pax_headers = {"comment": "abc123" + "0" * 34} + tarinfo.size = 0 + tar.addfile(tarinfo, io.BytesIO()) + + # Mock requests.get - must support context manager + mock_response = MagicMock() + mock_response.raise_for_status = Mock() + mock_response.raw = io.BytesIO(tar_bytes.getvalue()) + mock_response.__enter__ = Mock(return_value=mock_response) + mock_response.__exit__ = Mock(return_value=False) + + import requests + + monkeypatch.setattr(requests, "get", lambda *args, **kwargs: mock_response) + + # Test download + downloader = ArchiveDownloader("https://test.com/repo.tar.gz", {"Authorization": "Bearer test"}) + extracted_dir, commit_sha = downloader.download_and_extract(str(tmp_path)) + + # Verify extraction happened + assert path.isdir(extracted_dir) + # Verify SHA extracted + assert commit_sha == "abc123" + "0" * 34 + # Verify cleanup - archive file removed + assert not path.exists(path.join(str(tmp_path), "archive.tar.gz")) + + @staticmethod + def test_download_repos_signature(monkeypatch): + """Test download_repos returns correct format""" + + # Mock download_repo to return crafted SHAs + monkeypatch.setattr(rgs, "download_repo", lambda url, headers, extract_to: "crafted_sha_123") + + success, revisions = rgs.download_repos("/tmp/test") + + # Verify signature: (bool, (sha1, sha2)) + assert isinstance(success, bool) + assert success is True + assert isinstance(revisions, tuple) + assert len(revisions) == 2 + assert revisions[0] == "crafted_sha_123" + assert revisions[1] == "crafted_sha_123" + + @staticmethod + def test_download_repo_error_handling(tmp_path, monkeypatch): + """Test download_repo handles errors gracefully""" + from unittest.mock import Mock + + import requests + + # Mock requests.get to raise exception + monkeypatch.setattr(requests, "get", Mock(side_effect=requests.RequestException("Network error"))) + + result = rgs.download_repo("https://test.com/repo.tar.gz", {}, str(tmp_path)) + + # Should return None on error, not crash + assert result is None + + @staticmethod + def test_load_static_repo_fallback(tmp_path, monkeypatch): + """Test load_static_repo returns False when archive missing""" + monkeypatch.setattr(rgs, "STATIC_GITS_PATH", str(tmp_path)) + + # No archive file exists + result = rgs.load_static_repo(str(tmp_path / "insights_content_vulnerability_git")) + + # Should return False, not crash + assert result is False From 921ce22180719fc512b563d780d8f6055d15e61e Mon Sep 17 00:00:00 2001 From: Andrej Luptak Date: Thu, 10 Sep 2026 09:39:24 +0200 Subject: [PATCH 2/3] fix: gitlab links replacement --- Dockerfile | 10 +++++----- deploy/clowdapp.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 10d36d299..cbc8b8370 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ ARG ALT_REPO # Install PostgreSQL without pg_repack (will copy from builder) RUN (microdnf module enable -y postgresql:18 || curl -o /etc/yum.repos.d/postgresql.repo $ALT_REPO) && \ microdnf install -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \ - python312 libpq shadow-utils postgresql && \ + python312 libpq shadow-utils postgresql tar gzip && \ microdnf clean all # Copy pg_repack 1.5.2 from builder stage @@ -64,8 +64,8 @@ ENV MINIMAL_SCHEMA=172 # Baked-in content for FedRAMP ARG STATIC_ASSETS=0 ARG CONTENT_ARCHIVE_URL="https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz" -ARG PLAYBOOKS_ARCHIVE_URL="https://gitlab.cee.redhat.com/insights-rules/insights-playbooks/-/archive/master/insights-playbooks-master.tar.gz" -ARG VMAAS_ARCHIVE_URL="https://gitlab.cee.redhat.com/vmaas/vmaas-assets/-/archive/master/vmaas-assets-master.tar.gz" +ARG PLAYBOOKS_ARCHIVE_URL="https://github.com/RedHatInsights/insights-playbooks/archive/refs/heads/master.tar.gz" +ARG VMAAS_ARCHIVE_URL="https://github.com/RedHatInsights/vmaas-assets/archive/refs/heads/master.tar.gz" ARG ARCHIVE_EXT=".tar.gz" ARG GIT_TOKEN="" @@ -74,13 +74,13 @@ RUN if [ "${STATIC_ASSETS}" == 1 ] ; then \ update-ca-trust extract && \ \ # VMaaS assets - curl -L "${VMAAS_ARCHIVE_URL}" -o /tmp/vmaas${ARCHIVE_EXT} && \ + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${VMAAS_ARCHIVE_URL}" -o /tmp/vmaas${ARCHIVE_EXT} && \ tar -xzf /tmp/vmaas${ARCHIVE_EXT} -C /tmp && \ mv /tmp/vmaas-assets-* /engine/vmaas_assets_git && \ rm /tmp/vmaas${ARCHIVE_EXT} && \ \ # Insights playbooks, we need to keep the archive for sha computation - curl -L "${PLAYBOOKS_ARCHIVE_URL}" -o /engine/insights_playbooks_git${ARCHIVE_EXT} && \ + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${PLAYBOOKS_ARCHIVE_URL}" -o /engine/insights_playbooks_git${ARCHIVE_EXT} && \ tar -xzf /engine/insights_playbooks_git${ARCHIVE_EXT} -C /tmp && \ mv /tmp/insights-playbooks-* /engine/insights_playbooks_git && \ \ diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index e27997882..a84f586d6 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -1207,7 +1207,7 @@ parameters: value: https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz - name: PLAYBOOKS_ARCHIVE description: Archive URL for insights-playbooks archive download - value: https://gitlab.cee.redhat.com/insights-rules/insights-playbooks/-/archive/master/insights-playbooks-master.tar.gz + value: https://github.com/RedHatInsights/insights-playbooks/archive/refs/heads/master.tar.gz - name: SYSTEM_DELETION_THRESHOLD description: Delete systems marked as deleted before N hours value: "1" From 2a7aea756c6a0bc9a69cd0854cce8fc8e85fcbe5 Mon Sep 17 00:00:00 2001 From: Andrej Luptak Date: Thu, 10 Sep 2026 12:17:02 +0200 Subject: [PATCH 3/3] fix: remove archive ext and fix type anotation --- Dockerfile | 15 +++++++-------- taskomatic/jobs/archive_downloader.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index cbc8b8370..872015234 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,6 @@ ARG STATIC_ASSETS=0 ARG CONTENT_ARCHIVE_URL="https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz" ARG PLAYBOOKS_ARCHIVE_URL="https://github.com/RedHatInsights/insights-playbooks/archive/refs/heads/master.tar.gz" ARG VMAAS_ARCHIVE_URL="https://github.com/RedHatInsights/vmaas-assets/archive/refs/heads/master.tar.gz" -ARG ARCHIVE_EXT=".tar.gz" ARG GIT_TOKEN="" RUN if [ "${STATIC_ASSETS}" == 1 ] ; then \ @@ -74,19 +73,19 @@ RUN if [ "${STATIC_ASSETS}" == 1 ] ; then \ update-ca-trust extract && \ \ # VMaaS assets - curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${VMAAS_ARCHIVE_URL}" -o /tmp/vmaas${ARCHIVE_EXT} && \ - tar -xzf /tmp/vmaas${ARCHIVE_EXT} -C /tmp && \ + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${VMAAS_ARCHIVE_URL}" -o /tmp/vmaas.tar.gz && \ + tar -xzf /tmp/vmaas.tar.gz -C /tmp && \ mv /tmp/vmaas-assets-* /engine/vmaas_assets_git && \ - rm /tmp/vmaas${ARCHIVE_EXT} && \ + rm /tmp/vmaas.tar.gz && \ \ # Insights playbooks, we need to keep the archive for sha computation - curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${PLAYBOOKS_ARCHIVE_URL}" -o /engine/insights_playbooks_git${ARCHIVE_EXT} && \ - tar -xzf /engine/insights_playbooks_git${ARCHIVE_EXT} -C /tmp && \ + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${PLAYBOOKS_ARCHIVE_URL}" -o /engine/insights_playbooks_git.tar.gz && \ + tar -xzf /engine/insights_playbooks_git.tar.gz -C /tmp && \ mv /tmp/insights-playbooks-* /engine/insights_playbooks_git && \ \ # Insights content, we need to keep the archive for sha computation - curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${CONTENT_ARCHIVE_URL}" -o /engine/insights_content_vulnerability_git${ARCHIVE_EXT} && \ - tar -xzf /engine/insights_content_vulnerability_git${ARCHIVE_EXT} -C /tmp && \ + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${CONTENT_ARCHIVE_URL}" -o /engine/insights_content_vulnerability_git.tar.gz && \ + tar -xzf /engine/insights_content_vulnerability_git.tar.gz -C /tmp && \ mv /tmp/insights-content-vulnerability-* /engine/insights_content_vulnerability_git && \ \ echo "Downloaded static assets via HTTP" ; \ diff --git a/taskomatic/jobs/archive_downloader.py b/taskomatic/jobs/archive_downloader.py index 1795fca03..35fed9490 100644 --- a/taskomatic/jobs/archive_downloader.py +++ b/taskomatic/jobs/archive_downloader.py @@ -19,7 +19,7 @@ SUPPORTED_ARCHIVE_FORMATS = {".tar.gz": "tarball", ".zip": "zip"} -def _get_archive_type(archive_path: str) -> str: +def _get_archive_type(archive_path: str) -> str | None: """Detect archive type from path. Returns 'tarball', 'zip', or None""" for ext, archive_type in SUPPORTED_ARCHIVE_FORMATS.items(): if archive_path.endswith(ext):