diff --git a/.github/workflows/source-regressions.yml b/.github/workflows/source-regressions.yml index bdfa06df..c85205fa 100644 --- a/.github/workflows/source-regressions.yml +++ b/.github/workflows/source-regressions.yml @@ -6,6 +6,8 @@ on: paths: - nx2/nup.py - nx2/make_game.py + - nx2/host_cpu.py + - tests/test_cpu_target.py - nx2/build.py - nx2/unpack.py - nx2/test_nup_paths.py @@ -68,6 +70,8 @@ on: paths: - nx2/nup.py - nx2/make_game.py + - nx2/host_cpu.py + - tests/test_cpu_target.py - nx2/build.py - nx2/unpack.py - nx2/test_nup_paths.py @@ -162,6 +166,8 @@ jobs: run: python -I -B tests/test_cpm_discovery.py - name: Check a failed step's error line run: python -I -B tests/test_failure_reason.py + - name: Check the CPU target follows the CPU + run: python -I -B tests/test_cpu_target.py - name: C++ compiler for the import-hold check if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y g++ diff --git a/nx2/build.py b/nx2/build.py index 13974c5b..37c925d9 100644 --- a/nx2/build.py +++ b/nx2/build.py @@ -234,24 +234,7 @@ def ffmpeg_version_key(): # expression is rounded as written (the exact bodies and FPX1's proofs assume it). RX_MARCH=x86-64: the old baseline # (issue 363: a Xeon E5-2667 v2 (Ivy Bridge, no AVX2) built the game and then stopped at its first AVX2 instruction, # 0xc000001d. The game is built on the PC it runs on: a CPU without x86-64-v3's features gets x86-64-v2, or x86-64) -def host_march(): - want = ("avx2", "fma", "bmi2", "movbe") - try: - if os.name == "nt": - import ctypes - pf = ctypes.windll.kernel32.IsProcessorFeaturePresent - # (PF_AVX2_INSTRUCTIONS_AVAILABLE 40; PF_SSE4_2_INSTRUCTIONS_AVAILABLE 38. Windows reports no FMA/BMI2 bit, but - # every AVX2 CPU has them, as x86-64-v3 counts them together) - if pf(40): - return "x86-64-v3" - return "x86-64-v2" if pf(38) else "x86-64" - flags = open("/proc/cpuinfo").read().split("flags", 1)[1].split(chr(10), 1)[0].split() - if all(f in flags for f in want): - return "x86-64-v3" - return "x86-64-v2" if all(f in flags for f in ("sse4_2", "popcnt", "ssse3")) else "x86-64" - except Exception: - return "x86-64-v3" - +from host_cpu import host_march # (make_game.py asks it too: the AppImage's renderer) MARCH = os.environ.get("RX_MARCH") or host_march() if MARCH != "x86-64-v3": diff --git a/nx2/host_cpu.py b/nx2/host_cpu.py new file mode 100644 index 00000000..227a3c40 --- /dev/null +++ b/nx2/host_cpu.py @@ -0,0 +1,23 @@ +"""The x86-64 level this PC's CPU runs (nx2/build.py's -march; issue 363). The game is built on the PC it runs on, so a +CPU without x86-64-v3's features (AVX2, FMA, BMI2, MOVBE) gets x86-64-v2, or the x86-64 baseline. RX_MARCH, read by +build.py, still chooses.""" +import os + + +def host_march(): + want = ("avx2", "fma", "bmi2", "movbe") + try: + if os.name == "nt": + import ctypes + pf = ctypes.windll.kernel32.IsProcessorFeaturePresent + # (PF_AVX2_INSTRUCTIONS_AVAILABLE 40; PF_SSE4_2_INSTRUCTIONS_AVAILABLE 38. Windows reports no FMA/BMI2 bit, but + # every AVX2 CPU has them, as x86-64-v3 counts them together) + if pf(40): + return "x86-64-v3" + return "x86-64-v2" if pf(38) else "x86-64" + flags = open("/proc/cpuinfo").read().split("flags", 1)[1].split(chr(10), 1)[0].split() + if all(f in flags for f in want): + return "x86-64-v3" + return "x86-64-v2" if all(f in flags for f in ("sse4_2", "popcnt", "ssse3")) else "x86-64" + except Exception: + return "x86-64-v3" diff --git a/nx2/make_game.py b/nx2/make_game.py index 50df5030..6d7285bc 100644 --- a/nx2/make_game.py +++ b/nx2/make_game.py @@ -145,8 +145,23 @@ def check_cache_paths(out, paths): "Choose a different output folder." % path) +def check_cpu(): + """(linux) The AppImage's Vulkan back end (HOOKSHOT_VK_PREBUILT) is built for x86-64-v3 elsewhere + (scripts/linux/appimage.sh), while the game follows this PC (nx2/host_cpu.py): on a CPU without AVX2 it would stop at + its start. Said now, before the game is read, not 40 minutes in (issue 363)""" + if not os.environ.get("HOOKSHOT_VK_PREBUILT") or os.environ.get("RX_MARCH"): + return + sys.path.insert(0, HERE) + from host_cpu import host_march + m = host_march() + if m != "x86-64-v3": + fail("Your processor has no AVX2 (it runs %s), and the AppImage's renderer needs it. Build Hookshot from source " + "instead (see the readme): a source build follows your processor." % m) + + def check(a, work): check_windows_sdk() # (first: said in plain words when the game is chosen, not after it has been read) + check_cpu() if a.nup: probe = os.path.join(work, "probe") shutil.rmtree(probe, ignore_errors=True) @@ -486,6 +501,7 @@ def main(): os.makedirs(out, exist_ok=True) # (linux) the compiler probe waits until the compile. The game-free cache test stops at the lift, and a missing # compiler is still said before the long compile, not after the package has been read + check_cpu() # (no game read yet: the AppImage on a CPU without AVX2 stops here) if not LINUX: check_windows_sdk() drop_stale_objects(work) diff --git a/scripts/linux/appimage.sh b/scripts/linux/appimage.sh index cbf34387..78b5776d 100755 --- a/scripts/linux/appimage.sh +++ b/scripts/linux/appimage.sh @@ -22,7 +22,7 @@ PY=$DEPS/python/bin/python3 # ---- the Vulkan back end, built here once (it needs no game) ---- VKW=$B/vk-work rm -rf "$VKW/sa/build/hookshot_vk.so" -OOT_WORK=$VKW RX_VK_ONLY=1 RX_RELEASE=1 "$PY" -I nx2/build.py --standalone --jobs "${JOBS:-$(nproc)}" +OOT_WORK=$VKW RX_MARCH=x86-64-v3 RX_VK_ONLY=1 RX_RELEASE=1 "$PY" -I nx2/build.py --standalone --jobs "${JOBS:-$(nproc)}" VK=$VKW/sa/build/hookshot_vk.so [ -f "$VK" ] || { echo "hookshot_vk.so was not built"; exit 1; } diff --git a/tests/test_cpu_target.py b/tests/test_cpu_target.py new file mode 100644 index 00000000..c4d2f1ec --- /dev/null +++ b/tests/test_cpu_target.py @@ -0,0 +1,120 @@ +"""The game's CPU target follows the PC that builds it, and the AppImage says up front when its renderer cannot run +(nx2/host_cpu.py, nx2/make_game.py's check_cpu; issue 363). Game-free. + +python -I -B tests/test_cpu_target.py +""" +import builtins +import contextlib +import importlib.util +import io +import os +from pathlib import Path +import sys +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +host_cpu = load("host_cpu_under_test", ROOT / "nx2" / "host_cpu.py") + +# (/proc/cpuinfo flags, cut to what matters plus some noise) +HASWELL = "fpu sse sse2 pni ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt xsave avx f16c lahf_lm abm bmi1 avx2 bmi2" +IVY_BRIDGE = "fpu sse sse2 pni ssse3 cx16 sse4_1 sse4_2 popcnt xsave avx f16c lahf_lm" # (the Xeon E5-2667 v2 of 363) +CORE2 = "fpu sse sse2 pni ssse3 cx16 lahf_lm" + + +def cpuinfo(flags): + return "processor\t: 0\nvendor_id\t: GenuineIntel\nflags\t\t: %s\nvmx flags\t: ept\n\nprocessor\t: 1\n" \ + "flags\t\t: %s\n" % (flags, flags) + + +def march_on_linux(text): + real_open = builtins.open + + def fake_open(path, *args, **kwargs): + if path == "/proc/cpuinfo": + return io.StringIO(text) + return real_open(path, *args, **kwargs) + with mock.patch.object(host_cpu.os, "name", "posix"), mock.patch.object(builtins, "open", fake_open): + return host_cpu.host_march() + + +class HostMarch(unittest.TestCase): + def test_levels_from_cpuinfo(self): + self.assertEqual(march_on_linux(cpuinfo(HASWELL)), "x86-64-v3") + self.assertEqual(march_on_linux(cpuinfo(IVY_BRIDGE)), "x86-64-v2") + self.assertEqual(march_on_linux(cpuinfo(CORE2)), "x86-64") + + def test_unreadable_cpu_keeps_v3(self): + self.assertEqual(march_on_linux("processor\t: 0\n"), "x86-64-v3") + + def test_windows(self): + class K32: + def __init__(self, present): + self.IsProcessorFeaturePresent = lambda pf: pf in present + for present, want in (({38, 40}, "x86-64-v3"), ({38}, "x86-64-v2"), (set(), "x86-64")): + ctypes = mock.MagicMock() + ctypes.windll.kernel32 = K32(present) + with mock.patch.object(host_cpu.os, "name", "nt"), mock.patch.dict(sys.modules, {"ctypes": ctypes}): + self.assertEqual(host_cpu.host_march(), want, present) + + def test_build_and_appimage_use_it(self): + build = (ROOT / "nx2" / "build.py").read_text(encoding="utf-8") + self.assertIn("from host_cpu import host_march", build) + self.assertIn('MARCH = os.environ.get("RX_MARCH") or host_march()', build) + appimage = (ROOT / "scripts" / "linux" / "appimage.sh").read_text(encoding="utf-8") + self.assertIn("RX_MARCH=x86-64-v3 RX_VK_ONLY=1", appimage) # (its prebuilt renderer: not the CI runner's CPU) + + +class AppImageCheck(unittest.TestCase): + """make_game.py stops before reading the game when the AppImage's x86-64-v3 renderer meets a CPU without AVX2""" + + def setUp(self): + self.make_game = load("make_game_cpu_test", ROOT / "nx2" / "make_game.py") + + def run_check(self, env, march): + fake = mock.MagicMock(host_march=mock.MagicMock(return_value=march)) + with mock.patch.dict(os.environ, {}, clear=False), mock.patch.dict(sys.modules, {"host_cpu": fake}), \ + contextlib.redirect_stdout(io.StringIO()) as out: + for k in ("HOOKSHOT_VK_PREBUILT", "RX_MARCH"): + os.environ.pop(k, None) + os.environ.update(env) + try: + self.make_game.check_cpu() + except SystemExit as stopped: + return stopped.code, out.getvalue() + return None, out.getvalue() + + def test_appimage_without_avx2_stops(self): + code, out = self.run_check({"HOOKSHOT_VK_PREBUILT": "/x/hookshot_vk.so"}, "x86-64-v2") + self.assertEqual(code, 1) + self.assertIn("AVX2", out) + + def test_appimage_with_avx2_goes_on(self): + self.assertIsNone(self.run_check({"HOOKSHOT_VK_PREBUILT": "/x/hookshot_vk.so"}, "x86-64-v3")[0]) + + def test_source_build_without_avx2_goes_on(self): + self.assertIsNone(self.run_check({}, "x86-64-v2")[0]) + + def test_rx_march_is_the_players_choice(self): + env = {"HOOKSHOT_VK_PREBUILT": "/x/hookshot_vk.so", "RX_MARCH": "x86-64-v2"} + self.assertIsNone(self.run_check(env, "x86-64-v2")[0]) + + def test_checked_before_the_game_is_read(self): + src = (ROOT / "nx2" / "make_game.py").read_text(encoding="utf-8") + main = src[src.index("def main("):] + self.assertLess(main.index("check_cpu()"), main.index("drop_stale_objects(work)")) + check = src[src.index("def check(a, work):"):] + self.assertLess(check.index("check_cpu()"), check.index("if a.nup:")) + + +if __name__ == "__main__": + unittest.main()