diff --git a/gltest/direct/sdk_loader.py b/gltest/direct/sdk_loader.py index d463270..9b6743e 100644 --- a/gltest/direct/sdk_loader.py +++ b/gltest/direct/sdk_loader.py @@ -12,7 +12,6 @@ import shutil import tarfile import zipfile -import platform import tempfile import urllib.error import urllib.request @@ -24,27 +23,16 @@ GITHUB_API_RELEASES = "https://api.github.com/repos/genlayerlabs/genvm-manager/releases" -def _host_release_asset() -> str: - """genvm-manager (v0.6+) ships a whole-tree tarball per platform, named - genvm--.tar.xz — pick the one matching this host.""" - machine = platform.machine().lower() - arch = "arm64" if machine in ("aarch64", "arm64") else "amd64" - os_name = "macos" if platform.system().lower() == "darwin" else "linux" - return f"genvm-{arch}-{os_name}.tar.xz" - - -# Download candidates, newest-scheme first: the genvm-manager per-platform whole -# tree, then the pre-v0.6 runner-only bundles (genvm-runners-all → genvm-universal). -RUNNER_BUNDLE_ASSETS = ( - _host_release_asset(), - "genvm-runners-all.tar.xz", - "genvm-universal.tar.xz", -) +# Download candidates, newest-scheme first. These bundles contain the runner +# archives required by the direct loader. +RUNNER_BUNDLE_ASSETS = ("genvm-runners-all.tar.xz", "genvm-universal.tar.xz") GENVM_VERSION_ENV = "GENVM_VERSION" -FALLBACK_VERSION = "v0.6.0-rc0" +FALLBACK_VERSION = "v0.6.0-rc3" # v0.3 runner trees use .zip; the v0.2 legacy-runners tree uses .tar. RUNNER_ARCHIVE_EXTS = (".tar", ".zip") +BUNDLE_CACHE_DIR = CACHE_DIR / "bundles-v2" +TREE_CACHE_DIR = CACHE_DIR / "trees-v2" RUNNER_TYPE = "py-genlayer" STD_LIB_TYPE = "py-lib-genlayer-std" @@ -70,42 +58,70 @@ def parse_contract_header(contract_path: Path) -> Dict[str, str]: return deps +def _query_latest_version() -> Optional[str]: + """Newest suitable stable release, or newest RC when no stable exists.""" + req = urllib.request.Request( + f"{GITHUB_API_RELEASES}?per_page=100", + headers={ + "User-Agent": "gltest-direct", + "Accept": "application/vnd.github+json", + }, + ) + with urllib.request.urlopen(req, timeout=10) as resp: + releases = json.loads(resp.read().decode("utf-8")) + + prerelease_candidate = None + for release in releases: + if release.get("draft"): + continue + asset_names = {asset.get("name") for asset in release.get("assets", [])} + if not asset_names.intersection(RUNNER_BUNDLE_ASSETS): + continue + if release.get("prerelease"): + prerelease_candidate = prerelease_candidate or release["tag_name"] + continue + return release["tag_name"] + return prerelease_candidate + + def get_latest_version() -> str: - """Newest non-prerelease GenVM release that ships a known runner bundle.""" + """Return current published release, or the RC fallback offline.""" try: - req = urllib.request.Request( - f"{GITHUB_API_RELEASES}?per_page=100", - headers={ - "User-Agent": "gltest-direct", - "Accept": "application/vnd.github+json", - }, - ) - with urllib.request.urlopen(req, timeout=10) as resp: - releases = json.loads(resp.read().decode("utf-8")) - for release in releases: - if release.get("prerelease") or release.get("draft"): - continue - asset_names = {asset.get("name") for asset in release.get("assets", [])} - if asset_names.intersection(RUNNER_BUNDLE_ASSETS): - return release["tag_name"] + return _query_latest_version() or FALLBACK_VERSION except Exception as exc: print( f"Warning: could not resolve latest GenVM version ({exc}); " f"falling back to {FALLBACK_VERSION}", file=sys.stderr, ) - return FALLBACK_VERSION + return FALLBACK_VERSION def resolve_version() -> str: - """GenVM version to use: GENVM_VERSION env var > newest cached > latest release.""" + """Resolve GenVM version with explicit/current/offline-safe precedence. + + A cache is not authoritative: a stale cached RC must not shadow the + current manager release. The network-resolved release wins whenever it + differs from the offline fallback; cached artifacts are retained as an + offline fallback for development environments without network access. + """ pinned = os.environ.get(GENVM_VERSION_ENV) if pinned: return pinned + try: + latest = _query_latest_version() + if latest: + return latest + except Exception as exc: + print( + f"Warning: could not resolve latest GenVM version ({exc}); " + "checking the local cache", + file=sys.stderr, + ) cached = list_cached_versions() if cached: return cached[0] - return get_latest_version() + return FALLBACK_VERSION def _version_sort_key(version: str) -> tuple: @@ -115,11 +131,11 @@ def _version_sort_key(version: str) -> tuple: def list_cached_versions() -> List[str]: """List all cached genvm versions, newest first.""" - if not CACHE_DIR.exists(): + if not BUNDLE_CACHE_DIR.exists(): return [] versions = [] - for f in CACHE_DIR.glob("genvm-universal-*.tar.xz"): + for f in BUNDLE_CACHE_DIR.glob("genvm-universal-*.tar.xz"): match = re.search(r"genvm-universal-(.+)\.tar\.xz", f.name) if match: versions.append(match.group(1)) @@ -146,7 +162,11 @@ def _download_to(url: str, dest: Path) -> None: downloaded += len(chunk) if total: pct = downloaded * 100 // total - print(f"\r {pct}% ({downloaded // 1024 // 1024}MB)", end="", flush=True) + print( + f"\r {pct}% ({downloaded // 1024 // 1024}MB)", + end="", + flush=True, + ) tmp_path = tmp.name @@ -158,7 +178,8 @@ def download_artifacts(version: str) -> Path: """Download the GenVM runner bundle for version if not cached.""" CACHE_DIR.mkdir(parents=True, exist_ok=True) - tarball_path = CACHE_DIR / f"genvm-universal-{version}.tar.xz" + BUNDLE_CACHE_DIR.mkdir(parents=True, exist_ok=True) + tarball_path = BUNDLE_CACHE_DIR / f"genvm-universal-{version}.tar.xz" if tarball_path.exists(): return tarball_path @@ -179,6 +200,19 @@ def download_artifacts(version: str) -> Path: ) from last_error +def _extract_zip(archive: zipfile.ZipFile, destination: Path) -> None: + """Extract a runner zip without allowing members to escape the cache dir.""" + destination_root = destination.resolve() + for member in archive.infolist(): + member_path = (destination / member.filename).resolve() + if ( + member_path != destination_root + and destination_root not in member_path.parents + ): + raise ValueError(f"unsafe runner zip member: {member.filename}") + archive.extractall(destination) + + def _extract_local_runner( root: Path, runner_type: str, runner_hash: Optional[str] ) -> Path: @@ -199,17 +233,24 @@ def _extract_local_runner( raise FileNotFoundError(f"runner {runner_type}:{runner_hash} not under {root}") archive = hits[-1] dest = ( - CACHE_DIR / "extracted" / "local" / runner_type + CACHE_DIR + / "extracted" + / "local" + / runner_type / (archive.parent.name + archive.stem) ) if not dest.exists(): dest.mkdir(parents=True, exist_ok=True) - if archive.suffix == ".zip": - with zipfile.ZipFile(archive) as inner: - inner.extractall(dest) - else: - with tarfile.open(archive, "r:") as inner: - inner.extractall(dest, filter="data") + try: + if archive.suffix == ".zip": + with zipfile.ZipFile(archive) as inner: + _extract_zip(inner, dest) + else: + with tarfile.open(archive, "r:") as inner: + inner.extractall(dest, filter="data") + except Exception: + shutil.rmtree(dest, ignore_errors=True) + raise return dest @@ -220,12 +261,12 @@ def _extract_release_tree(tarball_path: Path, version: str) -> Path: runner-only bundle, so we unpack it to a directory that looks exactly like a GENVM_PREBUILT_DIR and then resolve runners through the same globbing path. """ - tree = CACHE_DIR / "trees" / version + tree = TREE_CACHE_DIR / version if (tree / ".extracted").exists(): return tree if tree.exists(): shutil.rmtree(tree) - trees = CACHE_DIR / "trees" + trees = TREE_CACHE_DIR trees.mkdir(parents=True, exist_ok=True) # Extract into a process-unique dir, mark it complete, then publish with an # atomic rename. Concurrent cold-cache extractions each use their own tmp and diff --git a/tests/gltest_direct/test_sdk_loader.py b/tests/gltest_direct/test_sdk_loader.py index dd28db5..9aad158 100644 --- a/tests/gltest_direct/test_sdk_loader.py +++ b/tests/gltest_direct/test_sdk_loader.py @@ -1,7 +1,11 @@ """Unit tests for direct-runner GenVM version and artifact resolution.""" +import io import json import urllib.error +import zipfile + +import pytest from gltest.direct import sdk_loader @@ -23,39 +27,58 @@ def read(self): class TestResolveVersion: - """resolve_version() precedence: env var > cache > latest release.""" + """resolve_version() precedence: explicit > current release > cache fallback.""" def test_env_var_takes_precedence(self, monkeypatch): monkeypatch.setenv(sdk_loader.GENVM_VERSION_ENV, "v1.2.3") monkeypatch.setattr(sdk_loader, "list_cached_versions", lambda: ["v0.2.16"]) - monkeypatch.setattr(sdk_loader, "get_latest_version", lambda: "v0.9.9") + monkeypatch.setattr(sdk_loader, "_query_latest_version", lambda: "v0.9.9") assert sdk_loader.resolve_version() == "v1.2.3" - def test_falls_back_to_newest_cached_version(self, monkeypatch): + def test_current_release_wins_over_stale_cached_version(self, monkeypatch): monkeypatch.delenv(sdk_loader.GENVM_VERSION_ENV, raising=False) monkeypatch.setattr(sdk_loader, "list_cached_versions", lambda: ["v0.2.16"]) - monkeypatch.setattr(sdk_loader, "get_latest_version", lambda: "v0.9.9") + monkeypatch.setattr(sdk_loader, "_query_latest_version", lambda: "v0.6.0-rc3") + + assert sdk_loader.resolve_version() == "v0.6.0-rc3" - assert sdk_loader.resolve_version() == "v0.2.16" + def test_offline_falls_back_to_newest_cached_version(self, monkeypatch, capsys): + monkeypatch.delenv(sdk_loader.GENVM_VERSION_ENV, raising=False) + monkeypatch.setattr(sdk_loader, "list_cached_versions", lambda: ["v0.6.0-rc0"]) + + def _offline(): + raise OSError("network down") + + monkeypatch.setattr(sdk_loader, "_query_latest_version", _offline) + + assert sdk_loader.resolve_version() == "v0.6.0-rc0" + assert "checking the local cache" in capsys.readouterr().err + + def test_offline_without_cache_uses_current_fallback(self, monkeypatch): + monkeypatch.delenv(sdk_loader.GENVM_VERSION_ENV, raising=False) + monkeypatch.setattr(sdk_loader, "list_cached_versions", lambda: []) + monkeypatch.setattr(sdk_loader, "_query_latest_version", lambda: None) + + assert sdk_loader.resolve_version() == "v0.6.0-rc3" def test_falls_back_to_latest_when_no_cache(self, monkeypatch): monkeypatch.delenv(sdk_loader.GENVM_VERSION_ENV, raising=False) monkeypatch.setattr(sdk_loader, "list_cached_versions", lambda: []) - monkeypatch.setattr(sdk_loader, "get_latest_version", lambda: "v0.9.9") + monkeypatch.setattr(sdk_loader, "_query_latest_version", lambda: "v0.9.9") assert sdk_loader.resolve_version() == "v0.9.9" class TestGetLatestVersion: - """get_latest_version() skips pre-releases and assetless releases.""" + """get_latest_version() selects current RCs and skips unusable releases.""" def test_skips_releases_without_a_bundle_asset(self, monkeypatch): releases = [ { - "tag_name": "v0.9.9", - "prerelease": False, - "assets": [{"name": "genvm-linux-amd64.tar.xz"}], + "tag_name": "v0.6.0-rc3", + "prerelease": True, + "assets": [{"name": sdk_loader.RUNNER_BUNDLE_ASSETS[0]}], }, { "tag_name": "v0.2.16", @@ -88,12 +111,18 @@ def test_accepts_renamed_runners_all_asset(self, monkeypatch): assert sdk_loader.get_latest_version() == "v0.3.0" - def test_skips_prereleases(self, monkeypatch): + def test_skips_drafts_and_assetless_releases(self, monkeypatch): releases = [ { - "tag_name": "v0.3.0-rc0", + "tag_name": "v0.6.0-rc4", "prerelease": True, - "assets": [{"name": "genvm-runners-all.tar.xz"}], + "draft": True, + "assets": [{"name": sdk_loader.RUNNER_BUNDLE_ASSETS[0]}], + }, + { + "tag_name": "v0.6.0-rc3", + "prerelease": True, + "assets": [], }, { "tag_name": "v0.2.16", @@ -107,6 +136,19 @@ def test_skips_prereleases(self, monkeypatch): assert sdk_loader.get_latest_version() == "v0.2.16" + def test_uses_rc_when_no_stable_bundle_exists(self, monkeypatch): + releases = [ + { + "tag_name": "v0.6.0-rc3", + "prerelease": True, + "assets": [{"name": sdk_loader.RUNNER_BUNDLE_ASSETS[0]}], + } + ] + monkeypatch.setattr( + "urllib.request.urlopen", lambda *a, **k: _FakeResponse(releases) + ) + assert sdk_loader.get_latest_version() == "v0.6.0-rc3" + def test_returns_fallback_when_api_unreachable(self, monkeypatch, capsys): def _boom(*a, **k): raise OSError("network down") @@ -121,12 +163,26 @@ class TestListCachedVersions: """list_cached_versions() orders versions numerically, newest first.""" def test_orders_versions_numerically(self, monkeypatch, tmp_path): - monkeypatch.setattr(sdk_loader, "CACHE_DIR", tmp_path) + monkeypatch.setattr(sdk_loader, "BUNDLE_CACHE_DIR", tmp_path / "bundles-v2") + sdk_loader.BUNDLE_CACHE_DIR.mkdir() for version in ("v0.2.9", "v0.2.16", "v0.10.0"): - (tmp_path / f"genvm-universal-{version}.tar.xz").write_bytes(b"") + ( + sdk_loader.BUNDLE_CACHE_DIR / f"genvm-universal-{version}.tar.xz" + ).write_bytes(b"") assert sdk_loader.list_cached_versions() == ["v0.10.0", "v0.2.16", "v0.2.9"] + def test_ignores_legacy_root_cache(self, monkeypatch, tmp_path): + monkeypatch.setattr(sdk_loader, "CACHE_DIR", tmp_path) + monkeypatch.setattr(sdk_loader, "BUNDLE_CACHE_DIR", tmp_path / "bundles-v2") + sdk_loader.BUNDLE_CACHE_DIR.mkdir() + (tmp_path / "genvm-universal-v0.6.0-rc0.tar.xz").write_bytes(b"legacy") + (sdk_loader.BUNDLE_CACHE_DIR / "genvm-universal-v0.6.0-rc3.tar.xz").write_bytes( + b"current" + ) + + assert sdk_loader.list_cached_versions() == ["v0.6.0-rc3"] + class TestDownloadArtifacts: """download_artifacts() resolves whichever bundle asset a release ships.""" @@ -137,6 +193,7 @@ def _http_404(url): def test_returns_cached_tarball_without_downloading(self, monkeypatch, tmp_path): monkeypatch.setattr(sdk_loader, "CACHE_DIR", tmp_path) + monkeypatch.setattr(sdk_loader, "BUNDLE_CACHE_DIR", tmp_path) cached = tmp_path / "genvm-universal-v0.2.16.tar.xz" cached.write_bytes(b"cached") @@ -149,6 +206,7 @@ def _fail(*a, **k): def test_uses_first_available_asset(self, monkeypatch, tmp_path): monkeypatch.setattr(sdk_loader, "CACHE_DIR", tmp_path) + monkeypatch.setattr(sdk_loader, "BUNDLE_CACHE_DIR", tmp_path) tried = [] def _download(url, dest): @@ -167,6 +225,7 @@ def _download(url, dest): def test_falls_back_to_old_asset_on_404(self, monkeypatch, tmp_path): monkeypatch.setattr(sdk_loader, "CACHE_DIR", tmp_path) + monkeypatch.setattr(sdk_loader, "BUNDLE_CACHE_DIR", tmp_path) tried = [] def _download(url, dest): @@ -187,6 +246,7 @@ def _download(url, dest): def test_raises_when_no_asset_found(self, monkeypatch, tmp_path): monkeypatch.setattr(sdk_loader, "CACHE_DIR", tmp_path) + monkeypatch.setattr(sdk_loader, "BUNDLE_CACHE_DIR", tmp_path) def _download(url, dest): raise self._http_404(url) @@ -199,3 +259,41 @@ def _download(url, dest): assert "v9.9.9" in str(e) else: raise AssertionError("expected FileNotFoundError") + + +class TestExtractLocalRunner: + """Runner ZIP extraction cannot escape or poison the local cache.""" + + @staticmethod + def _runner_zip(root, members): + archive = root / "runners" / "py-genlayer" / "ab" / "cd.zip" + archive.parent.mkdir(parents=True) + with zipfile.ZipFile(archive, mode="w") as bundle: + for name, content in members.items(): + bundle.writestr(name, content) + return archive + + def test_extracts_safe_runner_zip(self, monkeypatch, tmp_path): + root = tmp_path / "prebuilt" + self._runner_zip(root, {"runner.json": '{"Seq": []}'}) + monkeypatch.setattr(sdk_loader, "CACHE_DIR", tmp_path / "cache") + + extracted = sdk_loader._extract_local_runner(root, "py-genlayer", "abcd") + + assert (extracted / "runner.json").read_text() == '{"Seq": []}' + + def test_rejects_zip_traversal_and_removes_partial_cache( + self, monkeypatch, tmp_path + ): + root = tmp_path / "prebuilt" + self._runner_zip(root, {"../escape.py": "unsafe"}) + cache = tmp_path / "cache" + monkeypatch.setattr(sdk_loader, "CACHE_DIR", cache) + + with pytest.raises(ValueError, match="unsafe runner zip member"): + sdk_loader._extract_local_runner(root, "py-genlayer", "abcd") + + assert not (cache / "extracted" / "local" / "py-genlayer" / "abcd").exists() + assert not ( + cache / "extracted" / "local" / "py-genlayer" / "escape.py" + ).exists()