Add timeout and process cleanup to igor.exe invocations

classes/commands/IgorRunTestsCommand.py

@@ -61,6 +61,10 @@
 
 IGOR_PATH = IGOR_DIR / 'windows'/ 'x64' / 'igor.exe'
 
+# Bounds how long a single igor.exe invocation may run before it's considered hung
+# (e.g. stuck endlessly retrying a runtime download) and its process tree is killed.
+IGOR_TIMEOUT_SECONDS = 5 * 60
+
 BUILD_FILE_PATH = USER_DIR / "build.bff"
 
 SANDBOXED_PLATFORMS = ['windows', 'mac', 'linux']
@@ -481,7 +485,7 @@ def compare_versions(self, version_a, version_b):
     # Igor
 
     async def igor_get_license(self, access_key: str, output_path: Path):
-        await async_utils.run_and_capture(IGOR_PATH, [f'-ak={access_key}', f'-of={output_path}', 'Runtime', 'FetchLicense'])
+        await async_utils.run_and_capture(IGOR_PATH, [f'-ak={access_key}', f'-of={output_path}', 'Runtime', 'FetchLicense'], timeout=IGOR_TIMEOUT_SECONDS)
 
     async def igor_get_runtime_version(self, user_folder: Path, feed: str, expected_version: str):
         # This will prevent browser cache
@@ -492,7 +496,7 @@ async def igor_get_runtime_version(self, user_folder: Path, feed: str, expected_
             args.append(expected_version)
 
         # Execute command
-        result = await async_utils.run_and_capture(IGOR_PATH, args)
+        result = await async_utils.run_and_capture(IGOR_PATH, args, timeout=IGOR_TIMEOUT_SECONDS)
 
         pattern = re.compile(r'Version (\d+\.\d+\.\d+\.\d+)')
         match = pattern.search(result)
@@ -516,7 +520,7 @@ async def igor_install_runtime(self, user_folder: Path, feed: str, version: str,
         args = [f'/uf={user_folder}', f'/ru={feed}?cachebust={cacheBust}', f'/rp={RUNTIME_DIR}', f'/m={modules}', 'Runtime', 'Install', version]
 
         # Execute command
-        await async_utils.run_and_capture(IGOR_PATH, args)
+        await async_utils.run_and_capture(IGOR_PATH, args, timeout=IGOR_TIMEOUT_SECONDS)
 
         return RUNTIME_DIR / f'runtime-{version}'
 

utils/async_utils.py

@@ -167,21 +167,35 @@ async def check_keypress_win():
     else:
         await check_keypress_unix()
 
-async def run_and_capture(exe_path: str, args: list[str], extra_env: dict[str, str] | None = None):
+async def run_and_capture(exe_path: str, args: list[str], extra_env: dict[str, str] | None = None, timeout: float | None = None):
     # Create a stop event for capturing output
     stop_event = asyncio.Event()
 
     # Start the subprocess
     process = await run_exe(exe_path, args, extra_env=extra_env)
 
-    # Capture the output
-    stdout_output = await capture_output(process, stop_event, collect=True)
-
-    # Wait for the subprocess to exit
-    await process.wait()
-
-    # Ensure the stop event is set to clean up the capture task
-    stop_event.set()
+    try:
+        # Capture the output, bounded by timeout if provided so a hung/looping
+        # process (e.g. igor.exe endlessly retrying a download) fails fast
+        # instead of blocking until the outer CI job timeout kills things.
+        stdout_output = await asyncio.wait_for(capture_output(process, stop_event, collect=True), timeout=timeout)
+
+        # Wait for the subprocess to exit
+        await asyncio.wait_for(process.wait(), timeout=timeout)
+    except (asyncio.TimeoutError, asyncio.CancelledError) as e:
+        LOGGER.error(f"'{exe_path}' {'timed out after ' + str(timeout) + 's' if isinstance(e, asyncio.TimeoutError) else 'was cancelled'}; killing process tree (pid={process.pid})")
+        kill_process_tree(process.pid)
+        try:
+            await asyncio.wait_for(process.wait(), timeout=10)
+        except (asyncio.TimeoutError, ProcessLookupError):
+            pass
+
+        if isinstance(e, asyncio.TimeoutError):
+            raise TimeoutError(f"'{exe_path}' did not complete within {timeout} seconds") from e
+        raise
+    finally:
+        # Ensure the stop event is set to clean up the capture task
+        stop_event.set()
 
     LOGGER.info(f'Process completed')
     return stdout_output