AppImage: say up front when its renderer needs AVX2 the CPU lacks

.github/workflows/source-regressions.yml

@@ -6,6 +6,8 @@ on:
     paths:
       - nx2/nup.py
       - nx2/make_game.py
+      - nx2/host_cpu.py
+      - tests/test_cpu_target.py
       - nx2/build.py
       - nx2/unpack.py
       - nx2/test_nup_paths.py
@@ -68,6 +70,8 @@ on:
     paths:
       - nx2/nup.py
       - nx2/make_game.py
+      - nx2/host_cpu.py
+      - tests/test_cpu_target.py
       - nx2/build.py
       - nx2/unpack.py
       - nx2/test_nup_paths.py
@@ -162,6 +166,8 @@ jobs:
         run: python -I -B tests/test_cpm_discovery.py
       - name: Check a failed step's error line
         run: python -I -B tests/test_failure_reason.py
+      - name: Check the CPU target follows the CPU
+        run: python -I -B tests/test_cpu_target.py
       - name: C++ compiler for the import-hold check
         if: runner.os == 'Linux'
         run: sudo apt-get update && sudo apt-get install -y g++

nx2/build.py

@@ -234,24 +234,7 @@ def ffmpeg_version_key():
 # expression is rounded as written (the exact bodies and FPX1's proofs assume it). RX_MARCH=x86-64: the old baseline
 # (issue 363: a Xeon E5-2667 v2 (Ivy Bridge, no AVX2) built the game and then stopped at its first AVX2 instruction,
 # 0xc000001d. The game is built on the PC it runs on: a CPU without x86-64-v3's features gets x86-64-v2, or x86-64)
-def host_march():
-    want = ("avx2", "fma", "bmi2", "movbe")
-    try:
-        if os.name == "nt":
-            import ctypes
-            pf = ctypes.windll.kernel32.IsProcessorFeaturePresent
-            # (PF_AVX2_INSTRUCTIONS_AVAILABLE 40; PF_SSE4_2_INSTRUCTIONS_AVAILABLE 38. Windows reports no FMA/BMI2 bit, but
-            # every AVX2 CPU has them, as x86-64-v3 counts them together)
-            if pf(40):
-                return "x86-64-v3"
-            return "x86-64-v2" if pf(38) else "x86-64"
-        flags = open("/proc/cpuinfo").read().split("flags", 1)[1].split(chr(10), 1)[0].split()
-        if all(f in flags for f in want):
-            return "x86-64-v3"
-        return "x86-64-v2" if all(f in flags for f in ("sse4_2", "popcnt", "ssse3")) else "x86-64"
-    except Exception:
-        return "x86-64-v3"
-
+from host_cpu import host_march   # (make_game.py asks it too: the AppImage's renderer)
 
 MARCH = os.environ.get("RX_MARCH") or host_march()
 if MARCH != "x86-64-v3":

nx2/host_cpu.py

@@ -0,0 +1,23 @@
+"""The x86-64 level this PC's CPU runs (nx2/build.py's -march; issue 363). The game is built on the PC it runs on, so a
+CPU without x86-64-v3's features (AVX2, FMA, BMI2, MOVBE) gets x86-64-v2, or the x86-64 baseline. RX_MARCH, read by
+build.py, still chooses."""
+import os
+
+
+def host_march():
+    want = ("avx2", "fma", "bmi2", "movbe")
+    try:
+        if os.name == "nt":
+            import ctypes
+            pf = ctypes.windll.kernel32.IsProcessorFeaturePresent
+            # (PF_AVX2_INSTRUCTIONS_AVAILABLE 40; PF_SSE4_2_INSTRUCTIONS_AVAILABLE 38. Windows reports no FMA/BMI2 bit, but
+            # every AVX2 CPU has them, as x86-64-v3 counts them together)
+            if pf(40):
+                return "x86-64-v3"
+            return "x86-64-v2" if pf(38) else "x86-64"
+        flags = open("/proc/cpuinfo").read().split("flags", 1)[1].split(chr(10), 1)[0].split()
+        if all(f in flags for f in want):
+            return "x86-64-v3"
+        return "x86-64-v2" if all(f in flags for f in ("sse4_2", "popcnt", "ssse3")) else "x86-64"
+    except Exception:
+        return "x86-64-v3"

nx2/make_game.py

@@ -145,8 +145,23 @@ def check_cache_paths(out, paths):
                  "Choose a different output folder." % path)
 
 
+def check_cpu():
+    """(linux) The AppImage's Vulkan back end (HOOKSHOT_VK_PREBUILT) is built for x86-64-v3 elsewhere
+    (scripts/linux/appimage.sh), while the game follows this PC (nx2/host_cpu.py): on a CPU without AVX2 it would stop at
+    its start. Said now, before the game is read, not 40 minutes in (issue 363)"""
+    if not os.environ.get("HOOKSHOT_VK_PREBUILT") or os.environ.get("RX_MARCH"):
+        return
+    sys.path.insert(0, HERE)
+    from host_cpu import host_march
+    m = host_march()
+    if m != "x86-64-v3":
+        fail("Your processor has no AVX2 (it runs %s), and the AppImage's renderer needs it. Build Hookshot from source "
+             "instead (see the readme): a source build follows your processor." % m)
+
+
 def check(a, work):
     check_windows_sdk()   # (first: said in plain words when the game is chosen, not after it has been read)
+    check_cpu()
     if a.nup:
         probe = os.path.join(work, "probe")
         shutil.rmtree(probe, ignore_errors=True)
@@ -486,6 +501,7 @@ def main():
     os.makedirs(out, exist_ok=True)
     # (linux) the compiler probe waits until the compile. The game-free cache test stops at the lift, and a missing
     # compiler is still said before the long compile, not after the package has been read
+    check_cpu()   # (no game read yet: the AppImage on a CPU without AVX2 stops here)
     if not LINUX:
         check_windows_sdk()
     drop_stale_objects(work)

scripts/linux/appimage.sh

@@ -22,7 +22,7 @@ PY=$DEPS/python/bin/python3
 # ---- the Vulkan back end, built here once (it needs no game) ----
 VKW=$B/vk-work
 rm -rf "$VKW/sa/build/hookshot_vk.so"
-OOT_WORK=$VKW RX_VK_ONLY=1 RX_RELEASE=1 "$PY" -I nx2/build.py --standalone --jobs "${JOBS:-$(nproc)}"
+OOT_WORK=$VKW RX_MARCH=x86-64-v3 RX_VK_ONLY=1 RX_RELEASE=1 "$PY" -I nx2/build.py --standalone --jobs "${JOBS:-$(nproc)}"
 VK=$VKW/sa/build/hookshot_vk.so
 [ -f "$VK" ] || { echo "hookshot_vk.so was not built"; exit 1; }
 

tests/test_cpu_target.py

@@ -0,0 +1,120 @@
+"""The game's CPU target follows the PC that builds it, and the AppImage says up front when its renderer cannot run
+(nx2/host_cpu.py, nx2/make_game.py's check_cpu; issue 363). Game-free.
+
+python -I -B tests/test_cpu_target.py
+"""
+import builtins
+import contextlib
+import importlib.util
+import io
+import os
+from pathlib import Path
+import sys
+import unittest
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def load(name, path):
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+host_cpu = load("host_cpu_under_test", ROOT / "nx2" / "host_cpu.py")
+
+# (/proc/cpuinfo flags, cut to what matters plus some noise)
+HASWELL = "fpu sse sse2 pni ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt xsave avx f16c lahf_lm abm bmi1 avx2 bmi2"
+IVY_BRIDGE = "fpu sse sse2 pni ssse3 cx16 sse4_1 sse4_2 popcnt xsave avx f16c lahf_lm"   # (the Xeon E5-2667 v2 of 363)
+CORE2 = "fpu sse sse2 pni ssse3 cx16 lahf_lm"
+
+
+def cpuinfo(flags):
+    return "processor\t: 0\nvendor_id\t: GenuineIntel\nflags\t\t: %s\nvmx flags\t: ept\n\nprocessor\t: 1\n" \
+           "flags\t\t: %s\n" % (flags, flags)
+
+
+def march_on_linux(text):
+    real_open = builtins.open
+
+    def fake_open(path, *args, **kwargs):
+        if path == "/proc/cpuinfo":
+            return io.StringIO(text)
+        return real_open(path, *args, **kwargs)
+    with mock.patch.object(host_cpu.os, "name", "posix"), mock.patch.object(builtins, "open", fake_open):
+        return host_cpu.host_march()
+
+
+class HostMarch(unittest.TestCase):
+    def test_levels_from_cpuinfo(self):
+        self.assertEqual(march_on_linux(cpuinfo(HASWELL)), "x86-64-v3")
+        self.assertEqual(march_on_linux(cpuinfo(IVY_BRIDGE)), "x86-64-v2")
+        self.assertEqual(march_on_linux(cpuinfo(CORE2)), "x86-64")
+
+    def test_unreadable_cpu_keeps_v3(self):
+        self.assertEqual(march_on_linux("processor\t: 0\n"), "x86-64-v3")
+
+    def test_windows(self):
+        class K32:
+            def __init__(self, present):
+                self.IsProcessorFeaturePresent = lambda pf: pf in present
+        for present, want in (({38, 40}, "x86-64-v3"), ({38}, "x86-64-v2"), (set(), "x86-64")):
+            ctypes = mock.MagicMock()
+            ctypes.windll.kernel32 = K32(present)
+            with mock.patch.object(host_cpu.os, "name", "nt"), mock.patch.dict(sys.modules, {"ctypes": ctypes}):
+                self.assertEqual(host_cpu.host_march(), want, present)
+
+    def test_build_and_appimage_use_it(self):
+        build = (ROOT / "nx2" / "build.py").read_text(encoding="utf-8")
+        self.assertIn("from host_cpu import host_march", build)
+        self.assertIn('MARCH = os.environ.get("RX_MARCH") or host_march()', build)
+        appimage = (ROOT / "scripts" / "linux" / "appimage.sh").read_text(encoding="utf-8")
+        self.assertIn("RX_MARCH=x86-64-v3 RX_VK_ONLY=1", appimage)   # (its prebuilt renderer: not the CI runner's CPU)
+
+
+class AppImageCheck(unittest.TestCase):
+    """make_game.py stops before reading the game when the AppImage's x86-64-v3 renderer meets a CPU without AVX2"""
+
+    def setUp(self):
+        self.make_game = load("make_game_cpu_test", ROOT / "nx2" / "make_game.py")
+
+    def run_check(self, env, march):
+        fake = mock.MagicMock(host_march=mock.MagicMock(return_value=march))
+        with mock.patch.dict(os.environ, {}, clear=False), mock.patch.dict(sys.modules, {"host_cpu": fake}), \
+                contextlib.redirect_stdout(io.StringIO()) as out:
+            for k in ("HOOKSHOT_VK_PREBUILT", "RX_MARCH"):
+                os.environ.pop(k, None)
+            os.environ.update(env)
+            try:
+                self.make_game.check_cpu()
+            except SystemExit as stopped:
+                return stopped.code, out.getvalue()
+        return None, out.getvalue()
+
+    def test_appimage_without_avx2_stops(self):
+        code, out = self.run_check({"HOOKSHOT_VK_PREBUILT": "/x/hookshot_vk.so"}, "x86-64-v2")
+        self.assertEqual(code, 1)
+        self.assertIn("AVX2", out)
+
+    def test_appimage_with_avx2_goes_on(self):
+        self.assertIsNone(self.run_check({"HOOKSHOT_VK_PREBUILT": "/x/hookshot_vk.so"}, "x86-64-v3")[0])
+
+    def test_source_build_without_avx2_goes_on(self):
+        self.assertIsNone(self.run_check({}, "x86-64-v2")[0])
+
+    def test_rx_march_is_the_players_choice(self):
+        env = {"HOOKSHOT_VK_PREBUILT": "/x/hookshot_vk.so", "RX_MARCH": "x86-64-v2"}
+        self.assertIsNone(self.run_check(env, "x86-64-v2")[0])
+
+    def test_checked_before_the_game_is_read(self):
+        src = (ROOT / "nx2" / "make_game.py").read_text(encoding="utf-8")
+        main = src[src.index("def main("):]
+        self.assertLess(main.index("check_cpu()"), main.index("drop_stale_objects(work)"))
+        check = src[src.index("def check(a, work):"):]
+        self.assertLess(check.index("check_cpu()"), check.index("if a.nup:"))
+
+
+if __name__ == "__main__":
+    unittest.main()