chore: replace git dependency for http

Dockerfile

@@ -39,7 +39,7 @@
 # Install PostgreSQL without pg_repack (will copy from builder)
 RUN (microdnf module enable -y postgresql:18 || curl -o /etc/yum.repos.d/postgresql.repo $ALT_REPO) && \
     microdnf install -y --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \
-        python312 libpq shadow-utils git-core postgresql && \
+        python312 libpq shadow-utils postgresql tar gzip && \
     microdnf clean all
 
 # Copy pg_repack 1.5.2 from builder stage
@@ -63,18 +63,32 @@
 
 # Baked-in content for FedRAMP
 ARG STATIC_ASSETS=0
+ARG CONTENT_ARCHIVE_URL="https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz"
+ARG PLAYBOOKS_ARCHIVE_URL="https://github.com/RedHatInsights/insights-playbooks/archive/refs/heads/master.tar.gz"
+ARG VMAAS_ARCHIVE_URL="https://github.com/RedHatInsights/vmaas-assets/archive/refs/heads/master.tar.gz"
 ARG GIT_TOKEN=""
+
 RUN if [ "${STATIC_ASSETS}" == 1 ] ; then \
         curl -o /etc/pki/ca-trust/source/anchors/2022-IT-Root-CA.crt https://certs.corp.redhat.com/certs/2022-IT-Root-CA.pem && \
         update-ca-trust extract && \
-        git clone --depth 1 https://gitlab.cee.redhat.com/vmaas/vmaas-assets.git /engine/vmaas_assets_git && \
-        git clone --depth 1 https://gitlab.cee.redhat.com/insights-rules/insights-playbooks.git /engine/insights_playbooks_git && \
-        git clone --depth 1 "https://$GIT_TOKEN@github.com/RedHatInsights/insights-content-vulnerability.git" /engine/insights_content_vulnerability_git && \
-        # below is needed to avoid git 'detected dubious ownership' error when running as a rootless container...
-        git config --system --add safe.directory /engine/vmaas_assets_git && \
-        git config --system --add safe.directory /engine/insights_playbooks_git && \
-        git config --system --add safe.directory /engine/insights_content_vulnerability_git && \
-        echo "Cloned static assets" ; \
+        \
+        # VMaaS assets
+        curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${VMAAS_ARCHIVE_URL}" -o /tmp/vmaas.tar.gz && \
+        tar -xzf /tmp/vmaas.tar.gz -C /tmp && \
+        mv /tmp/vmaas-assets-* /engine/vmaas_assets_git && \
+        rm /tmp/vmaas.tar.gz && \
+        \
+        # Insights playbooks, we need to keep the archive for sha computation
+        curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${PLAYBOOKS_ARCHIVE_URL}" -o /engine/insights_playbooks_git.tar.gz && \
+        tar -xzf /engine/insights_playbooks_git.tar.gz -C /tmp && \
+        mv /tmp/insights-playbooks-* /engine/insights_playbooks_git && \
+        \
+        # Insights content, we need to keep the archive for sha computation
+        curl -L -H "Authorization: Bearer ${GIT_TOKEN}" "${CONTENT_ARCHIVE_URL}" -o /engine/insights_content_vulnerability_git.tar.gz && \
+        tar -xzf /engine/insights_content_vulnerability_git.tar.gz -C /tmp && \
+        mv /tmp/insights-content-vulnerability-* /engine/insights_content_vulnerability_git && \
+        \
+        echo "Downloaded static assets via HTTP" ; \
     fi
 
 USER insights

common/config.py

@@ -101,8 +101,8 @@ def __init__(self):
 
         # taskomatic jobs
         self.system_deletion_threshold = int(os.getenv("SYSTEM_DELETION_THRESHOLD", "24"))  # 24 hours
-        self.content_git_repo = os.getenv("CONTENT_GIT_REPO", "")
-        self.playbooks_git_repo = os.getenv("PLAYBOOKS_GIT_REPO", "")
+        self.content_archive_url = os.getenv("CONTENT_ARCHIVE_URL", "")
+        self.playbooks_archive_url = os.getenv("PLAYBOOKS_ARCHIVE_URL", "")
         self.git_token = os.getenv("GIT_TOKEN", "")  # used also by exploit-sync
         self.jobs = os.getenv("JOBS", "").split(",")
         self.jobs_startup = os.getenv("JOBS_STARTUP", "").split(",")

conf/taskomatic.env

@@ -1,8 +1,8 @@
 JOBS=stale_systems:5,delete_systems:30,rules_git_sync:240,db_metrics:15,cacheman:5,delete_notifications:720,migrate_data:999999
 JOBS_STARTUP=db_metrics,cacheman,migrate_data
 SYSTEM_DELETION_THRESHOLD=1
-CONTENT_GIT_REPO=
-PLAYBOOKS_GIT_REPO=
+CONTENT_ARCHIVE_URL=
+PLAYBOOKS_ARCHIVE_URL=
 ACCOUNTS_BLACKLIST=
 CACHE_MINIMAL_ACCOUNT_SYSTEMS=400
 CACHE_KEEP_KEEPALIVE_HOURS=168

deploy/clowdapp.yaml

@@ -225,10 +225,10 @@ objects:
           value: ${JOBS}
         - name: JOBS_STARTUP
           value: ${JOBS_STARTUP}
-        - name: CONTENT_GIT_REPO
-          value: ${CONTENT_GIT}
-        - name: PLAYBOOKS_GIT_REPO
-          value: https://github.com/RedHatInsights/insights-playbooks.git
+        - name: CONTENT_ARCHIVE_URL
+          value: ${CONTENT_ARCHIVE}
+        - name: PLAYBOOKS_ARCHIVE_URL
+          value: ${PLAYBOOKS_ARCHIVE}
         - name: GIT_TOKEN
           valueFrom:
             secretKeyRef:
@@ -1202,9 +1202,12 @@ parameters:
 - name: JOBS_STARTUP
   description: Jobs to run on app startup (restart)
   value: "db_metrics,cacheman,migrate_data"
-- name: CONTENT_GIT
-  description: Location of insights-content(-vulnerability) repo
-  value: https://github.com/RedHatInsights/insights-content-vulnerability.git
+- name: CONTENT_ARCHIVE
+  description: Archive URL for insights-content-vulnerability archive download
+  value: https://github.com/RedHatInsights/insights-content-vulnerability/archive/refs/heads/master.tar.gz
+- name: PLAYBOOKS_ARCHIVE
+  description: Archive URL for insights-playbooks archive download
+  value: https://github.com/RedHatInsights/insights-playbooks/archive/refs/heads/master.tar.gz
 - name: SYSTEM_DELETION_THRESHOLD
   description: Delete systems marked as deleted before N hours
   value: "1"

pyproject.toml

@@ -31,7 +31,6 @@ botocore = "^1.34.122"
 # Pinned due to build issue in Konflux
 cramjam = "=2.10.0"
 flask = "^3.1.1"
-gitpython = "^3.1.43"
 connexion = {version = "^3.1.0", extras = ["swagger-ui", "flask"]}
 gunicorn = "^23.0.0"
 peewee = "^3.18.1"
@@ -76,6 +75,7 @@ python-box = "^7.0.0"
 insights-core = "^3.3.2"
 pytest-cov = "^7.0.0"
 pytest-aiohttp = "^1.0.5"
+gitpython = "^3.1.43" # Tests use this
 
 [build-system]
 requires = ["poetry-core>=2.4.1", "setuptools-rust==1.11.1", "puccinialin==0.1"]

requirements-build.txt

@@ -531,7 +531,6 @@ setuptools==84.0.0 \
     #   charset-normalizer
     #   cryptography
     #   frozenlist
-    #   gitpython
     #   googleapis-common-protos
     #   grpcio
     #   gunicorn

requirements-dev.txt

@@ -580,6 +580,12 @@ frozenlist==1.8.0 ; python_version >= "3.12" and python_version < "3.14" \
     --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \
     --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \
     --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd
+gitdb==4.0.12 ; python_version >= "3.12" and python_version < "3.14" \
+    --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \
+    --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf
+gitpython==3.1.62 ; python_version >= "3.12" and python_version < "3.14" \
+    --hash=sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af \
+    --hash=sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e
 identify==2.6.19 ; python_version >= "3.12" and python_version < "3.14" \
     --hash=sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a \
     --hash=sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842
@@ -1221,6 +1227,9 @@ scramp==1.4.17 ; python_version >= "3.12" and python_version < "3.14" \
 six==1.17.0 ; python_version >= "3.12" and python_version < "3.14" \
     --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
     --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
+smmap==5.0.3 ; python_version >= "3.12" and python_version < "3.14" \
+    --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \
+    --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f
 testing-common-database==2.0.3 ; python_version >= "3.12" and python_version < "3.14" \
     --hash=sha256:965d80b2985315325dc358c3061b174a712f4d4d5bf6a80b58b11f9a1dd86d73 \
     --hash=sha256:e3ed492bf480a87f271f74c53b262caf5d85c8bc09989a8f534fa2283ec52492

requirements.txt

@@ -891,12 +891,6 @@ frozenlist==1.8.0 ; python_version >= "3.12" and python_version < "3.14" \
     --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \
     --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \
     --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd
-gitdb==4.0.12 ; python_version >= "3.12" and python_version < "3.14" \
-    --hash=sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571 \
-    --hash=sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf
-gitpython==3.1.62 ; python_version >= "3.12" and python_version < "3.14" \
-    --hash=sha256:1791de66309bc0c7cfca40bf8d2e3de7ca091cbf94e6051be1ad0722c61062af \
-    --hash=sha256:7002251225e10e29d2e1f49e6532613fe5d5d9f0b6f1f02997a52b38fe56899e
 google-auth==2.58.0 ; python_version >= "3.12" and python_version < "3.14" \
     --hash=sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd \
     --hash=sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d
@@ -1855,9 +1849,6 @@ semver==3.0.4 ; python_version >= "3.12" and python_version < "3.14" \
 six==1.17.0 ; python_version >= "3.12" and python_version < "3.14" \
     --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
     --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
-smmap==5.0.3 ; python_version >= "3.12" and python_version < "3.14" \
-    --hash=sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c \
-    --hash=sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f
 starlette==1.6.0 ; python_version >= "3.12" and python_version < "3.14" \
     --hash=sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c \
     --hash=sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b

taskomatic/jobs/archive_downloader.py

@@ -0,0 +1,106 @@
+"""Generic HTTP archive downloader."""
+
+import hashlib
+import os
+import shutil
+import tarfile
+import zipfile
+from pathlib import Path
+from typing import Tuple
+from urllib.parse import urlparse
+
+import requests
+
+from common.logging import get_logger
+
+LOGGER = get_logger(__name__)
+
+# Supported archive formats (order matters for detection)
+SUPPORTED_ARCHIVE_FORMATS = {".tar.gz": "tarball", ".zip": "zip"}
+
+
+def _get_archive_type(archive_path: str) -> str | None:
+    """Detect archive type from path. Returns 'tarball', 'zip', or None"""
+    for ext, archive_type in SUPPORTED_ARCHIVE_FORMATS.items():
+        if archive_path.endswith(ext):
+            return archive_type
+    return None
+
+
+def extract_commit_sha_from_archive(archive_path: str) -> str:
+    """
+    Extract commit SHA from local archive file.
+    Returns commit SHA (from PAX headers/ZIP comment, or SHA256 hash as fallback)
+    """
+    archive_type = _get_archive_type(archive_path)
+
+    try:
+        if archive_type == "zip":
+            with zipfile.ZipFile(archive_path, "r") as z:
+                if sha := z.comment.decode("utf-8").strip():
+                    LOGGER.debug("Extracted commit SHA from ZIP comment: %s", sha[:8])
+                    return sha
+        elif archive_type == "tarball":
+            with tarfile.open(archive_path, "r:gz") as t:
+                if (first := next(iter(t), None)) and first.pax_headers:
+                    if sha := first.pax_headers.get("comment"):
+                        LOGGER.debug("Extracted commit SHA from PAX headers: %s", sha[:8])
+                        return sha
+        else:
+            LOGGER.error("Unsupported archive format %s", archive_path)
+    except Exception as err:
+        LOGGER.warning("Could not extract commit SHA: %s", err)
+
+    # Fallback: SHA256 hash of archive
+    LOGGER.warning("No metadata SHA, using SHA256 hash of archive")
+    with open(archive_path, "rb") as f:
+        return hashlib.sha256(f.read()).hexdigest()[:40]
+
+
+class ArchiveDownloader:
+    """Generic downloader for tar.gz and zip archives"""
+
+    def __init__(self, archive_url: str, auth_headers: dict = None):
+        self.archive_url = archive_url
+        self.auth_headers = auth_headers or {}
+
+    def _extract_archive(self, archive_path: str, extract_to: str) -> None:
+        archive_type = _get_archive_type(archive_path)
+
+        if archive_type == "zip":
+            with zipfile.ZipFile(archive_path, "r") as z:
+                z.extractall(extract_to)
+        elif archive_type == "tarball":
+            with tarfile.open(archive_path, "r:gz") as t:
+                t.extractall(extract_to, filter="data")
+        else:
+            raise ValueError(f"Unsupported archive format: {archive_path}")
+
+    def download_and_extract(self, extract_to: str) -> Tuple[str, str]:
+        """Downloads, extracts, and returns (extracted_dir_path, commit_sha)"""
+        os.makedirs(extract_to, exist_ok=True)
+        # Detect extension from URL path (handles query params, supports multi-suffix like .tar.gz)
+        url_path = urlparse(self.archive_url).path
+        ext = "".join(Path(url_path).suffixes).lstrip(".")
+        archive_path = os.path.join(extract_to, f"archive.{ext}")
+
+        LOGGER.info("Downloading %s", self.archive_url)
+        with requests.get(self.archive_url, headers=self.auth_headers, stream=True, timeout=300) as r:
+            r.raise_for_status()
+            with open(archive_path, "wb") as f:
+                # tell Python to stream data securely from the source to drive
+                shutil.copyfileobj(r.raw, f)
+
+        commit_sha = extract_commit_sha_from_archive(archive_path)
+
+        self._extract_archive(archive_path, extract_to)
+        os.remove(archive_path)
+
+        # Unpacked archive may have a folder with a different naming inside, find the newly extracted folder
+        dirs = [os.path.join(extract_to, d) for d in os.listdir(extract_to) if os.path.isdir(os.path.join(extract_to, d))]
+        if not dirs:  # We should always have top level directory, so no fallback here
+            raise RuntimeError(f"Archive extraction failed: no directories found in {extract_to}")
+        extracted_dir = dirs[0]
+
+        LOGGER.debug("Extracted to: %s (SHA: %s)", extracted_dir, commit_sha[:8])
+        return extracted_dir, commit_sha