feat: lazy git clones via artifactory shim

mama/_version.py

@@ -1,2 +1,2 @@
 # this is parsed by pyproject.toml and defines current mamabuild version
-__version__ = "0.11.33"
+__version__ = "0.12.01"

mama/artifactory.py

@@ -10,7 +10,7 @@
 from .types.asset import Asset
 from .utils.system import Color, System, console, error
 import mama.package as package
-from .util import download_file, normalized_join, try_unzip
+from .util import download_file, normalized_join, try_unzip, is_network_error
 from .papa_deploy import PapaFileInfo
 
 
@@ -170,6 +170,7 @@ def artifactory_upload(ftp:ftplib.FTP_TLS, target_name:str, file_path:str):
     size = os.path.getsize(file_path)
     transferred = 0
     lastpercent = 0
+    indent = f'  - {target_name: <16} '
     with open(file_path, 'rb') as f:
         def print_progress(bytes):
             nonlocal transferred, lastpercent, size
@@ -180,16 +181,16 @@ def print_progress(bytes):
                 n = int(percent / 2)
                 left = '=' * n
                 right = ' ' * int(50 - n)
-                print(f'\r    |{left}>{right}| {percent:>3} %', end='')
-        print(f'    |>{" ":50}| {0:>3} %', end='')
+                console(f'\r{indent}|{left}>{right}| {percent:>3} %', end='')
+        console(f'{indent}|>{" ":50}| {0:>3} %', end='')
         # chdir into FTP_ROOT/target_name/
         try:
             ftp.cwd(target_name)
         except:
             ftp.mkd(target_name) # create subdirectory if needed
             ftp.cwd(target_name)
         ftp.storbinary(f'STOR {os.path.basename(file_path)}', f, callback=print_progress)
-        print(f'\r    |{"="*50}>| 100 %')
+        console(f'\r{indent}|{"="*50}>| 100 %')
 
 
 def artifact_already_exists(ftp:ftplib.FTP_TLS, target:BuildTarget, file_path:str):
@@ -270,11 +271,16 @@ def artifactory_load_target(target:BuildTarget, deploy_path, num_files_copied) -
 
 
 def _fetch_package(target:BuildTarget, url, archive, cache_dir):
+    if not target.config.is_network_available():
+        return None
     remote_file = f'http://{url}/{target.name}/{archive}.zip'
     try:
-        return download_file(remote_file, cache_dir, force=True, 
-                             message=f'    Artifactory fetch {url}/{archive} ')
+        return download_file(remote_file, cache_dir, force=True,
+                             message=f'  - {target.name: <16} Artifactory fetch {url}/{archive} ',
+                             name=target.name)
     except Exception as e:
+        if is_network_error(e):
+            target.config.mark_network_unavailable()
         if target.config.verbose or target.config.force_artifactory:
             error(f'    Artifactory fetch failed with {e} {url}/{archive}.zip')
 
@@ -317,7 +323,7 @@ def artifactory_fetch_and_reconfigure(target:BuildTarget) -> Tuple[bool, list]:
     archive = artifactory_archive_name(target)
     if not archive:
         return (False, None)
-    
+
     cache_dir = target.dep.dep_dir #target.dep.workspace
     local_file = normalized_join(cache_dir, f'{archive}.zip')
 
@@ -334,5 +340,67 @@ def artifactory_fetch_and_reconfigure(target:BuildTarget) -> Tuple[bool, list]:
     local_file = _fetch_package(target, url, archive, cache_dir)
     if not local_file:
         return (False, None)
-    console(f'    Artifactory unzip {archive}')
+    console(f'  - {target.name: <16} Artifactory unzip {archive}')
     return unzip_and_load_target(target, local_file)
+
+
+def try_load_artifactory_shim(dep) -> Tuple:
+    """
+    Probe artifactory for a prebuilt package using the commit hash resolved via
+    `git ls-remote` (no clone). On hit, construct a default BuildTarget, load
+    papa.txt exports/deps into it, write the shim marker, and return the target
+    plus its child dep_sources.
+
+    On miss (or when artifactory is not configured), returns (None, None) and
+    leaves dep state untouched so the caller can fall back to the clone path.
+
+    Returns (target_or_None, dep_sources_or_None).
+    """
+    from .build_target import BuildTarget  # local import to avoid cycle
+
+    config = dep.config
+    if not config.artifactory_ftp:
+        return (None, None)
+    if not dep.dep_source.is_git:
+        return (None, None)
+
+    git: Git = dep.dep_source
+
+    # Resolve commit hash without cloning. `init_commit_hash` already supports
+    # ls-remote and respects the stored git_status cache when `update` is not set.
+    commit_hash = git.init_commit_hash(dep, use_cache=True, fetch_remote=True)
+    if not commit_hash:
+        if config.verbose:
+            console(f'    {dep.name}  shim probe: could not resolve commit hash', color=Color.YELLOW)
+        return (None, None)
+    git.commit_hash = commit_hash  # cache for downstream consumers
+
+    # First probe: commit-hash-based archive name. Works for the common case.
+    probe_target = BuildTarget(name=dep.name, config=config, dep=dep, args=dep.target_args)
+    fetched, dependencies = artifactory_fetch_and_reconfigure(probe_target)
+
+    # Fallback: dep may pin target.version (e.g. boost 1.60), so its archive
+    # name doesn't include the commit hash. Sparse-fetch only the mamafile,
+    # grep self.version, and re-probe with that version.
+    if not fetched:
+        version = git.fetch_self_version_from_remote(dep)
+        if version:
+            if config.verbose:
+                console(f'    {dep.name}  shim probe: retrying with self.version={version}', color=Color.YELLOW)
+            probe_target = BuildTarget(name=dep.name, config=config, dep=dep, args=dep.target_args)
+            probe_target.version = version
+            fetched, dependencies = artifactory_fetch_and_reconfigure(probe_target)
+
+    if not fetched:
+        # Reset any side effect on the dep so the clone path can run cleanly.
+        dep.from_artifactory = False
+        return (None, None)
+
+    # Hit: persist marker and return the configured target.
+    archive = artifactory_archive_name(probe_target)
+    dep.write_shim_marker(archive_name=archive or '', commit_hash=commit_hash)
+    config.update_stats.record_shim()
+    if config.print:
+        console(f'  - Target {dep.name: <16} SHIM FETCHED {archive}', color=Color.GREEN)
+
+    return (probe_target, dependencies)

mama/build_config.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-import os, sys, tempfile, platform, psutil, shutil
+import os, sys, tempfile, platform, psutil, shutil, threading, time
 from typing import List, TYPE_CHECKING
 from mama.platforms.oclea import Oclea
 from mama.platforms.xilinx import Xilinx
@@ -17,6 +17,54 @@
 if TYPE_CHECKING:
     from .build_dependency import BuildDependency
 
+class UpdateStats:
+    """Counts and times clone/pull/shim-fetch activity during the load phase."""
+    def __init__(self):
+        self._lock = threading.Lock()
+        self.cloned = 0
+        self.pulled = 0
+        self.shim_fetched = 0
+        self._start = None
+        self._duration = 0.0
+
+    def start(self):
+        self._start = time.monotonic()
+
+    def stop(self):
+        if self._start is not None:
+            self._duration = time.monotonic() - self._start
+            self._start = None
+
+    def record_clone(self):
+        with self._lock: self.cloned += 1
+
+    def record_pull(self):
+        with self._lock: self.pulled += 1
+
+    def record_shim(self):
+        with self._lock: self.shim_fetched += 1
+
+    @property
+    def total(self) -> int:
+        return self.cloned + self.pulled + self.shim_fetched
+
+    @property
+    def duration(self) -> float:
+        return self._duration
+
+    def summary_line(self) -> str:
+        """One-line summary, or '' if nothing happened."""
+        if self.total == 0:
+            return ''
+        parts = []
+        if self.shim_fetched: parts.append(f'{self.shim_fetched} shim-fetched')
+        if self.pulled:       parts.append(f'{self.pulled} pulled')
+        if self.cloned:       parts.append(f'{self.cloned} cloned')
+        # Local import to avoid circular dependency with util
+        from .util import get_time_str
+        return f'Updated {self.total} target(s): {", ".join(parts)} in {get_time_str(self._duration)}'
+
+
 ###
 # Mama Build Configuration is created only once in the root project working directory
 # This configuration is then passed down to dependencies
@@ -54,6 +102,7 @@ def __init__(self, args):
         self.sanitize  = None # gcc/clang: -fsanitize=[thread|leak|address|undefined]
         self.coverage  = None # gcc/clang: gcov | msvc: /fsanitize-coverage=edge
         self.coverage_report = None # runs gcovr to generate coverage report
+        self.update_stats = UpdateStats() # clone/pull/shim counters for the load phase summary
         self.enable_clang_tidy = False # enables clang-tidy static analysis during build
         self.clang_tidy_path = None # resolved path to clang-tidy executable
         # supported platforms
@@ -124,6 +173,7 @@ def __init__(self, args):
             self.workspaces_root = util.normalized_path(os.getenv('HOMEPATH'))
         else:
             self.workspaces_root = os.getenv('HOME')
+        self._network_available = None  # None=untested, True/False=result
         self.unused_args = []
         self.loaded_dependencies : dict[str, BuildDependency] = {}
         self.parse_args(args)
@@ -1184,3 +1234,14 @@ def no_specific_target(self) -> bool:
         """ True if no target or 'all' was specified from cmdline """
         return self.no_target() or self.targets_all()
 
+    def is_network_available(self) -> bool:
+        """Lazily cached: True until a clearly network-related failure marks it False."""
+        return self._network_available is not False
+
+    def mark_network_unavailable(self):
+        if self._network_available is not False:
+            if self.print:
+                from .utils.system import console, Color
+                console('  Network unavailable — using cached packages where possible', color=Color.YELLOW)
+            self._network_available = False
+