diff --git a/Dockerfile b/Dockerfile index 491918415..872015234 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ ARG ALT_REPO # Install PostgreSQL without pg_repack (will copy from builder) RUN (microdnf module enable -y postgresql:18 || curl -o /etc/yum.repos.d/postgresql.repo $ALT_REPO) && \ microdnf install -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \ - python312 libpq shadow-utils git-core postgresql && \ + python312 libpq shadow-utils postgresql tar gzip && \ microdnf clean all # Copy pg_repack 1.5.2 from builder stage @@ -63,18 +63,32 @@ ENV MINIMAL_SCHEMA=172 # Baked-in content for FedRAMP ARG STATIC_ASSETS=0 +ARG CONTENT_ARCHIVE_URL="https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz" +ARG PLAYBOOKS_ARCHIVE_URL="https://github.com/RedHatInsights/insights-playbooks/archive/refs/heads/master.tar.gz" +ARG VMAAS_ARCHIVE_URL="https://github.com/RedHatInsights/vmaas-assets/archive/refs/heads/master.tar.gz" ARG GIT_TOKEN="" + RUN if [ "${STATIC_ASSETS}" == 1 ] ; then \ curl -o /etc/pki/ca-trust/source/anchors/2022-IT-Root-CA.crt https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem && \ update-ca-trust extract && \ - git clone --depth 1 https://gitlab.cee.redhat.com/vmaas/vmaas-assets.git /engine/vmaas_assets_git && \ - git clone --depth 1 https://gitlab.cee.redhat.com/insights-rules/insights-playbooks.git /engine/insights_playbooks_git && \ - git clone --depth 1 "https://$GIT_TOKEN@github.com/RedHatInsights/insights-content-vulnerability.git" /engine/insights_content_vulnerability_git && \ - # below is needed to avoid git 'detected dubious ownership' error when running as a rootless container... - git config --system --add safe.directory /engine/vmaas_assets_git && \ - git config --system --add safe.directory /engine/insights_playbooks_git && \ - git config --system --add safe.directory /engine/insights_content_vulnerability_git && \ - echo "Cloned static assets" ; \ + \ + # VMaaS assets + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${VMAAS_ARCHIVE_URL}" -o /tmp/vmaas.tar.gz && \ + tar -xzf /tmp/vmaas.tar.gz -C /tmp && \ + mv /tmp/vmaas-assets-* /engine/vmaas_assets_git && \ + rm /tmp/vmaas.tar.gz && \ + \ + # Insights playbooks, we need to keep the archive for sha computation + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${PLAYBOOKS_ARCHIVE_URL}" -o /engine/insights_playbooks_git.tar.gz && \ + tar -xzf /engine/insights_playbooks_git.tar.gz -C /tmp && \ + mv /tmp/insights-playbooks-* /engine/insights_playbooks_git && \ + \ + # Insights content, we need to keep the archive for sha computation + curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${CONTENT_ARCHIVE_URL}" -o /engine/insights_content_vulnerability_git.tar.gz && \ + tar -xzf /engine/insights_content_vulnerability_git.tar.gz -C /tmp && \ + mv /tmp/insights-content-vulnerability-* /engine/insights_content_vulnerability_git && \ + \ + echo "Downloaded static assets via HTTP" ; \ fi USER insights diff --git a/common/config.py b/common/config.py index 0b2901858..cfe9131de 100644 --- a/common/config.py +++ b/common/config.py @@ -101,8 +101,8 @@ def __init__(self): # taskomatic jobs self.system_deletion_threshold = int(os.getenv("SYSTEM_DELETION_THRESHOLD", "24")) # 24 hours - self.content_git_repo = os.getenv("CONTENT_GIT_REPO", "") - self.playbooks_git_repo = os.getenv("PLAYBOOKS_GIT_REPO", "") + self.content_archive_url = os.getenv("CONTENT_ARCHIVE_URL", "") + self.playbooks_archive_url = os.getenv("PLAYBOOKS_ARCHIVE_URL", "") self.git_token = os.getenv("GIT_TOKEN", "") # used also by exploit-sync self.jobs = os.getenv("JOBS", "").split(",") self.jobs_startup = os.getenv("JOBS_STARTUP", "").split(",") diff --git a/conf/taskomatic.env b/conf/taskomatic.env index 4dd3875e6..23d366d58 100644 --- a/conf/taskomatic.env +++ b/conf/taskomatic.env @@ -1,8 +1,8 @@ JOBS=stale_systems:5,delete_systems:30,rules_git_sync:240,db_metrics:15,cacheman:5,delete_notifications:720,migrate_data:999999 JOBS_STARTUP=db_metrics,cacheman,migrate_data SYSTEM_DELETION_THRESHOLD=1 -CONTENT_GIT_REPO= -PLAYBOOKS_GIT_REPO= +CONTENT_ARCHIVE_URL= +PLAYBOOKS_ARCHIVE_URL= ACCOUNTS_BLACKLIST= CACHE_MINIMAL_ACCOUNT_SYSTEMS=400 CACHE_KEEP_KEEPALIVE_HOURS=168 diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 1ef52994e..a84f586d6 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -225,10 +225,10 @@ objects: value: ${JOBS} - name: JOBS_STARTUP value: ${JOBS_STARTUP} - - name: CONTENT_GIT_REPO - value: ${CONTENT_GIT} - - name: PLAYBOOKS_GIT_REPO - value: https://github.com/RedHatInsights/insights-playbooks.git + - name: CONTENT_ARCHIVE_URL + value: ${CONTENT_ARCHIVE} + - name: PLAYBOOKS_ARCHIVE_URL + value: ${PLAYBOOKS_ARCHIVE} - name: GIT_TOKEN valueFrom: secretKeyRef: @@ -1202,9 +1202,12 @@ parameters: - name: JOBS_STARTUP description: Jobs to run on app startup (restart) value: "db_metrics,cacheman,migrate_data" -- name: CONTENT_GIT - description: Location of insights-content(-vulnerability) repo - value: https://github.com/RedHatInsights/insights-content-vulnerability.git +- name: CONTENT_ARCHIVE + description: Archive URL for insights-content-vulnerability archive download + value: https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz +- name: PLAYBOOKS_ARCHIVE + description: Archive URL for insights-playbooks archive download + value: https://github.com/RedHatInsights/insights-playbooks/archive/refs/heads/master.tar.gz - name: SYSTEM_DELETION_THRESHOLD description: Delete systems marked as deleted before N hours value: "1" diff --git a/poetry.lock b/poetry.lock index 8bab44639..c76ce2e0e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1499,7 +1499,7 @@ version = "4.0.12" description = "Git Object Database" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, @@ -1514,7 +1514,7 @@ version = "3.1.62" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "gitpython-3.1.62-py3-none-any.whl", hash = "sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e"}, {file = "gitpython-3.1.62.tar.gz", hash = "sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af"}, @@ -3824,7 +3824,7 @@ version = "5.0.3" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f"}, {file = "smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c"}, @@ -4323,4 +4323,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.14" -content-hash = "31d13d686be3946ce7cf476d620e2b70ec61101fd98b231a279f2e76660de5e0" +content-hash = "74255851a12804a03629ad0ca2c417280a2cc2b7b2a1a27ffbb99ed675fc6361" diff --git a/pyproject.toml b/pyproject.toml index a1007984a..d0607f985 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ botocore = "^1.34.122" # Pinned due to build issue in Konflux cramjam = "=2.10.0" flask = "^3.1.1" -gitpython = "^3.1.43" connexion = {version = "^3.1.0", extras = ["swagger-ui", "flask"]} gunicorn = "^23.0.0" peewee = "^3.18.1" @@ -76,6 +75,7 @@ python-box = "^7.0.0" insights-core = "^3.3.2" pytest-cov = "^7.0.0" pytest-aiohttp = "^1.0.5" +gitpython = "^3.1.43" # Tests use this [build-system] requires = ["poetry-core>=2.4.1", "setuptools-rust==1.11.1", "puccinialin==0.1"] diff --git a/requirements-build.txt b/requirements-build.txt index 8449ccbb3..632490139 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -531,7 +531,6 @@ setuptools==84.0.0 \ # charset-normalizer # cryptography # frozenlist - # gitpython # googleapis-common-protos # grpcio # gunicorn diff --git a/requirements-dev.txt b/requirements-dev.txt index 95cb060c5..52a4a9ecf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -580,6 +580,12 @@ frozenlist==1.8.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd +gitdb==4.0.12 ; python_version >= "3.12" and python_version < "3.14" \ + --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \ + --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf +gitpython==3.1.62 ; python_version >= "3.12" and python_version < "3.14" \ + --hash=sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af \ + --hash=sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e identify==2.6.19 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a \ --hash=sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842 @@ -1221,6 +1227,9 @@ scramp==1.4.17 ; python_version >= "3.12" and python_version < "3.14" \ six==1.17.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 +smmap==5.0.3 ; python_version >= "3.12" and python_version < "3.14" \ + --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \ + --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f testing-common-database==2.0.3 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:965d80b2985315325dc358c3061b174a712f4d4d5bf6a80b58b11f9a1dd86d73 \ --hash=sha256:e3ed492bf480a87f271f74c53b262caf5d85c8bc09989a8f534fa2283ec52492 diff --git a/requirements.txt b/requirements.txt index ba899fcc4..482989584 100644 --- a/requirements.txt +++ b/requirements.txt @@ -891,12 +891,6 @@ frozenlist==1.8.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd -gitdb==4.0.12 ; python_version >= "3.12" and python_version < "3.14" \ - --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \ - --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf -gitpython==3.1.62 ; python_version >= "3.12" and python_version < "3.14" \ - --hash=sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af \ - --hash=sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e google-auth==2.58.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd \ --hash=sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d @@ -1855,9 +1849,6 @@ semver==3.0.4 ; python_version >= "3.12" and python_version < "3.14" \ six==1.17.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 -smmap==5.0.3 ; python_version >= "3.12" and python_version < "3.14" \ - --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \ - --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f starlette==1.6.0 ; python_version >= "3.12" and python_version < "3.14" \ --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \ --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b diff --git a/taskomatic/jobs/archive_downloader.py b/taskomatic/jobs/archive_downloader.py new file mode 100644 index 000000000..35fed9490 --- /dev/null +++ b/taskomatic/jobs/archive_downloader.py @@ -0,0 +1,106 @@ +"""Generic HTTP archive downloader.""" + +import hashlib +import os +import shutil +import tarfile +import zipfile +from pathlib import Path +from typing import Tuple +from urllib.parse import urlparse + +import requests + +from common.logging import get_logger + +LOGGER = get_logger(__name__) + +# Supported archive formats (order matters for detection) +SUPPORTED_ARCHIVE_FORMATS = {".tar.gz": "tarball", ".zip": "zip"} + + +def _get_archive_type(archive_path: str) -> str | None: + """Detect archive type from path. Returns 'tarball', 'zip', or None""" + for ext, archive_type in SUPPORTED_ARCHIVE_FORMATS.items(): + if archive_path.endswith(ext): + return archive_type + return None + + +def extract_commit_sha_from_archive(archive_path: str) -> str: + """ + Extract commit SHA from local archive file. + Returns commit SHA (from PAX headers/ZIP comment, or SHA256 hash as fallback) + """ + archive_type = _get_archive_type(archive_path) + + try: + if archive_type == "zip": + with zipfile.ZipFile(archive_path, "r") as z: + if sha := z.comment.decode("utf-8").strip(): + LOGGER.debug("Extracted commit SHA from ZIP comment: %s", sha[:8]) + return sha + elif archive_type == "tarball": + with tarfile.open(archive_path, "r:gz") as t: + if (first := next(iter(t), None)) and first.pax_headers: + if sha := first.pax_headers.get("comment"): + LOGGER.debug("Extracted commit SHA from PAX headers: %s", sha[:8]) + return sha + else: + LOGGER.error("Unsupported archive format %s", archive_path) + except Exception as err: + LOGGER.warning("Could not extract commit SHA: %s", err) + + # Fallback: SHA256 hash of archive + LOGGER.warning("No metadata SHA, using SHA256 hash of archive") + with open(archive_path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest()[:40] + + +class ArchiveDownloader: + """Generic downloader for tar.gz and zip archives""" + + def __init__(self, archive_url: str, auth_headers: dict = None): + self.archive_url = archive_url + self.auth_headers = auth_headers or {} + + def _extract_archive(self, archive_path: str, extract_to: str) -> None: + archive_type = _get_archive_type(archive_path) + + if archive_type == "zip": + with zipfile.ZipFile(archive_path, "r") as z: + z.extractall(extract_to) + elif archive_type == "tarball": + with tarfile.open(archive_path, "r:gz") as t: + t.extractall(extract_to, filter="data") + else: + raise ValueError(f"Unsupported archive format: {archive_path}") + + def download_and_extract(self, extract_to: str) -> Tuple[str, str]: + """Downloads, extracts, and returns (extracted_dir_path, commit_sha)""" + os.makedirs(extract_to, exist_ok=True) + # Detect extension from URL path (handles query params, supports multi-suffix like .tar.gz) + url_path = urlparse(self.archive_url).path + ext = "".join(Path(url_path).suffixes).lstrip(".") + archive_path = os.path.join(extract_to, f"archive.{ext}") + + LOGGER.info("Downloading %s", self.archive_url) + with requests.get(self.archive_url, headers=self.auth_headers, stream=True, timeout=300) as r: + r.raise_for_status() + with open(archive_path, "wb") as f: + # tell Python to stream data securely from the source to drive + shutil.copyfileobj(r.raw, f) + + commit_sha = extract_commit_sha_from_archive(archive_path) + + self._extract_archive(archive_path, extract_to) + os.remove(archive_path) + + # Unpacked archive may have a folder with a different naming inside, find the newly extracted folder + dirs = [os.path.join(extract_to, d) for d in os.listdir(extract_to) if os.path.isdir(os.path.join(extract_to, d))] + if not dirs: # We should always have top level directory, so no fallback here + raise RuntimeError(f"Archive extraction failed: no directories found in {extract_to}") + extracted_dir = dirs[0] + + LOGGER.debug("Extracted to: %s (SHA: %s)", extracted_dir, commit_sha[:8]) + return extracted_dir, commit_sha diff --git a/taskomatic/jobs/rules_git_sync.py b/taskomatic/jobs/rules_git_sync.py index 65cde4571..6aa72ba4f 100755 --- a/taskomatic/jobs/rules_git_sync.py +++ b/taskomatic/jobs/rules_git_sync.py @@ -4,19 +4,22 @@ """ import os +import shutil import tempfile from datetime import datetime from functools import reduce +from typing import Tuple import yaml -from git import Repo -from git.cmd import GitCommandError from psycopg2.extras import execute_values from common.config import Config from common.logging import get_logger from common.logging import init_logging from common.utils import EXCEPTION_COUNT +from taskomatic.jobs.archive_downloader import SUPPORTED_ARCHIVE_FORMATS +from taskomatic.jobs.archive_downloader import ArchiveDownloader +from taskomatic.jobs.archive_downloader import extract_commit_sha_from_archive from taskomatic.jobs.common import get_conn LOGGER = get_logger(__name__) @@ -29,7 +32,7 @@ def store_versions(conn, content_version, playbooks_version): """ - Store SHA of latest imported commit and URL of git repositories used + Store SHA of latest imported commit and URL of git archive repositories used Args: content_version str: sha of HEAD of insights-content-vulnerability @@ -44,9 +47,9 @@ def store_versions(conn, content_version, playbooks_version): insights_playbooks_repo = %(playbooks_repo)s""", { "content_version": content_version, - "content_repo": CFG.content_git_repo, + "content_repo": CFG.content_archive_url, "playbooks_version": playbooks_version, - "playbooks_repo": CFG.playbooks_git_repo, + "playbooks_repo": CFG.playbooks_archive_url, }, ) @@ -520,37 +523,22 @@ def sync(tmpdirname: str, content_version: str, playbooks_version: str): # pyli conn.close() -def clone_repo(git_repo, repo_dir_name): - """Clone single git repo""" - if not git_repo: - LOGGER.error("Git repo for %s not set.", repo_dir_name) - return False - git_repo = git_repo.replace("github.com", "%s@github.com" % CFG.git_token) if CFG.git_token else git_repo - try: - repo = Repo.clone_from(git_repo, repo_dir_name, depth=1) - except GitCommandError: - LOGGER.exception("Error cloning repo: ") - return False - return repo.head.object.hexsha - - -def clone_repos(tmpdirname): - """Clone git repos to temp dir""" - revisions = ( - clone_repo(CFG.content_git_repo, f"{tmpdirname}/{CONTENT_GIT_NAME}"), - clone_repo(CFG.playbooks_git_repo, f"{tmpdirname}/{PLAYBOOKS_GIT_NAME}"), - ) - return all(revisions), revisions +def load_static_repo(repo_dir_name): + """Get version of static repo (FedRAMP mode)""" + repo_name = os.path.basename(repo_dir_name) + # Try all supported archive formats + for ext in SUPPORTED_ARCHIVE_FORMATS.keys(): + archive_path = os.path.join(STATIC_GITS_PATH, f"{repo_name}{ext}") + if os.path.exists(archive_path): + try: + return extract_commit_sha_from_archive(archive_path) + except Exception: + LOGGER.exception("Error loading repo SHA from archive: %s", archive_path) + return False -def load_static_repo(repo_dir_name): - """Get revision of single git repo""" - try: - repo = Repo(repo_dir_name) - except GitCommandError: - LOGGER.exception("Error loading repo: ") - return False - return repo.head.object.hexsha + LOGGER.error("No archive found for %s in %s", repo_name, STATIC_GITS_PATH) + return False def load_static_repos(dirname): @@ -562,15 +550,49 @@ def load_static_repos(dirname): return all(revisions), revisions +def download_repo(archive_url: str, auth_headers: dict, target_dir: str) -> str | None: + """Download repository using HTTP archive""" + if not archive_url: + LOGGER.error("Archive URL not provided.") + return None + + try: + # Use tmp dir for download and extract + # (to prevent nested directories and file rewriting during concurent download) + with tempfile.TemporaryDirectory() as dl_tmp: + downloader = ArchiveDownloader(archive_url, auth_headers) + extracted_path, commit_sha = downloader.download_and_extract(dl_tmp) + shutil.move(extracted_path, target_dir) + + return commit_sha + except Exception as err: + LOGGER.exception("Error downloading repo: %s", err) + from common.utils import EXCEPTION_COUNT + + EXCEPTION_COUNT.inc() + return None + + +def download_repos(tmpdirname: str) -> Tuple[bool, Tuple[str, str]]: + """Download content and playbooks repos""" + auth_headers = {"Authorization": f"Bearer {CFG.git_token}"} if CFG.git_token.strip() else {} + + content_sha = download_repo(CFG.content_archive_url, auth_headers, f"{tmpdirname}/{CONTENT_GIT_NAME}") + playbooks_sha = download_repo(CFG.playbooks_archive_url, auth_headers, f"{tmpdirname}/{PLAYBOOKS_GIT_NAME}") + + revisions = (content_sha, playbooks_sha) + return all(revisions), revisions + + def run(): """Application entrypoint""" LOGGER.info("Started rules_git_sync job.") if not CFG.is_fedramp: with tempfile.TemporaryDirectory() as tmpdirname: - LOGGER.info("Cloning repos.") - success, revisions = clone_repos(tmpdirname) + LOGGER.info("Downloading repos via HTTP.") + success, revisions = download_repos(tmpdirname) if success: - LOGGER.info("Cloning repos successful, starting sync.") + LOGGER.info("Download successful, starting sync.") sync(tmpdirname, revisions[0], revisions[1]) else: LOGGER.info("FedRAMP env, using static repositories.") diff --git a/tests/taskomatic_tests/test_rules_git_sync.py b/tests/taskomatic_tests/test_rules_git_sync.py index 566b52606..26ae8b9fb 100644 --- a/tests/taskomatic_tests/test_rules_git_sync.py +++ b/tests/taskomatic_tests/test_rules_git_sync.py @@ -59,3 +59,86 @@ def test_sync(pg_db_conn, monkeypatch): # pylint: disable=unused-argument assert row[1] is None cur.close() + + @staticmethod + def test_download_and_extract(tmp_path, monkeypatch): + """Test ArchiveDownloader downloads, extracts and cleans up""" + import io + import tarfile + from unittest.mock import MagicMock + from unittest.mock import Mock + + from taskomatic.jobs.archive_downloader import ArchiveDownloader + + # Create tarball in memory + tar_bytes = io.BytesIO() + with tarfile.open(fileobj=tar_bytes, mode="w:gz", format=tarfile.PAX_FORMAT) as tar: + tarinfo = tarfile.TarInfo(name="test-repo-main/README.md") + tarinfo.pax_headers = {"comment": "abc123" + "0" * 34} + tarinfo.size = 0 + tar.addfile(tarinfo, io.BytesIO()) + + # Mock requests.get - must support context manager + mock_response = MagicMock() + mock_response.raise_for_status = Mock() + mock_response.raw = io.BytesIO(tar_bytes.getvalue()) + mock_response.__enter__ = Mock(return_value=mock_response) + mock_response.__exit__ = Mock(return_value=False) + + import requests + + monkeypatch.setattr(requests, "get", lambda *args, **kwargs: mock_response) + + # Test download + downloader = ArchiveDownloader("https://test.com/repo.tar.gz", {"Authorization": "Bearer test"}) + extracted_dir, commit_sha = downloader.download_and_extract(str(tmp_path)) + + # Verify extraction happened + assert path.isdir(extracted_dir) + # Verify SHA extracted + assert commit_sha == "abc123" + "0" * 34 + # Verify cleanup - archive file removed + assert not path.exists(path.join(str(tmp_path), "archive.tar.gz")) + + @staticmethod + def test_download_repos_signature(monkeypatch): + """Test download_repos returns correct format""" + + # Mock download_repo to return crafted SHAs + monkeypatch.setattr(rgs, "download_repo", lambda url, headers, extract_to: "crafted_sha_123") + + success, revisions = rgs.download_repos("/tmp/test") + + # Verify signature: (bool, (sha1, sha2)) + assert isinstance(success, bool) + assert success is True + assert isinstance(revisions, tuple) + assert len(revisions) == 2 + assert revisions[0] == "crafted_sha_123" + assert revisions[1] == "crafted_sha_123" + + @staticmethod + def test_download_repo_error_handling(tmp_path, monkeypatch): + """Test download_repo handles errors gracefully""" + from unittest.mock import Mock + + import requests + + # Mock requests.get to raise exception + monkeypatch.setattr(requests, "get", Mock(side_effect=requests.RequestException("Network error"))) + + result = rgs.download_repo("https://test.com/repo.tar.gz", {}, str(tmp_path)) + + # Should return None on error, not crash + assert result is None + + @staticmethod + def test_load_static_repo_fallback(tmp_path, monkeypatch): + """Test load_static_repo returns False when archive missing""" + monkeypatch.setattr(rgs, "STATIC_GITS_PATH", str(tmp_path)) + + # No archive file exists + result = rgs.load_static_repo(str(tmp_path / "insights_content_vulnerability_git")) + + # Should return False, not crash + assert result is False