diff --git a/classes/commands/IgorRunTestsCommand.py b/classes/commands/IgorRunTestsCommand.py index a33586bc..99905b54 100644 --- a/classes/commands/IgorRunTestsCommand.py +++ b/classes/commands/IgorRunTestsCommand.py @@ -61,6 +61,10 @@ IGOR_PATH = IGOR_DIR / 'windows'/ 'x64' / 'igor.exe' +# Bounds how long a single igor.exe invocation may run before it's considered hung +# (e.g. stuck endlessly retrying a runtime download) and its process tree is killed. +IGOR_TIMEOUT_SECONDS = 5 * 60 + BUILD_FILE_PATH = USER_DIR / "build.bff" SANDBOXED_PLATFORMS = ['windows', 'mac', 'linux'] @@ -481,7 +485,7 @@ def compare_versions(self, version_a, version_b): # Igor async def igor_get_license(self, access_key: str, output_path: Path): - await async_utils.run_and_capture(IGOR_PATH, [f'-ak={access_key}', f'-of={output_path}', 'Runtime', 'FetchLicense']) + await async_utils.run_and_capture(IGOR_PATH, [f'-ak={access_key}', f'-of={output_path}', 'Runtime', 'FetchLicense'], timeout=IGOR_TIMEOUT_SECONDS) async def igor_get_runtime_version(self, user_folder: Path, feed: str, expected_version: str): # This will prevent browser cache @@ -492,7 +496,7 @@ async def igor_get_runtime_version(self, user_folder: Path, feed: str, expected_ args.append(expected_version) # Execute command - result = await async_utils.run_and_capture(IGOR_PATH, args) + result = await async_utils.run_and_capture(IGOR_PATH, args, timeout=IGOR_TIMEOUT_SECONDS) pattern = re.compile(r'Version (\d+\.\d+\.\d+\.\d+)') match = pattern.search(result) @@ -516,7 +520,7 @@ async def igor_install_runtime(self, user_folder: Path, feed: str, version: str, args = [f'/uf={user_folder}', f'/ru={feed}?cachebust={cacheBust}', f'/rp={RUNTIME_DIR}', f'/m={modules}', 'Runtime', 'Install', version] # Execute command - await async_utils.run_and_capture(IGOR_PATH, args) + await async_utils.run_and_capture(IGOR_PATH, args, timeout=IGOR_TIMEOUT_SECONDS) return RUNTIME_DIR / f'runtime-{version}' diff --git a/utils/async_utils.py b/utils/async_utils.py index 3eaa4d7e..8fdc1e88 100644 --- a/utils/async_utils.py +++ b/utils/async_utils.py @@ -167,21 +167,35 @@ async def check_keypress_win(): else: await check_keypress_unix() -async def run_and_capture(exe_path: str, args: list[str], extra_env: dict[str, str] | None = None): +async def run_and_capture(exe_path: str, args: list[str], extra_env: dict[str, str] | None = None, timeout: float | None = None): # Create a stop event for capturing output stop_event = asyncio.Event() # Start the subprocess process = await run_exe(exe_path, args, extra_env=extra_env) - # Capture the output - stdout_output = await capture_output(process, stop_event, collect=True) - - # Wait for the subprocess to exit - await process.wait() - - # Ensure the stop event is set to clean up the capture task - stop_event.set() + try: + # Capture the output, bounded by timeout if provided so a hung/looping + # process (e.g. igor.exe endlessly retrying a download) fails fast + # instead of blocking until the outer CI job timeout kills things. + stdout_output = await asyncio.wait_for(capture_output(process, stop_event, collect=True), timeout=timeout) + + # Wait for the subprocess to exit + await asyncio.wait_for(process.wait(), timeout=timeout) + except (asyncio.TimeoutError, asyncio.CancelledError) as e: + LOGGER.error(f"'{exe_path}' {'timed out after ' + str(timeout) + 's' if isinstance(e, asyncio.TimeoutError) else 'was cancelled'}; killing process tree (pid={process.pid})") + kill_process_tree(process.pid) + try: + await asyncio.wait_for(process.wait(), timeout=10) + except (asyncio.TimeoutError, ProcessLookupError): + pass + + if isinstance(e, asyncio.TimeoutError): + raise TimeoutError(f"'{exe_path}' did not complete within {timeout} seconds") from e + raise + finally: + # Ensure the stop event is set to clean up the capture task + stop_event.set() LOGGER.info(f'Process completed') return stdout_output