diff --git a/classes/commands/IgorRunTestsCommand.py b/classes/commands/IgorRunTestsCommand.py index 8664a25f..0223353d 100644 --- a/classes/commands/IgorRunTestsCommand.py +++ b/classes/commands/IgorRunTestsCommand.py @@ -327,7 +327,7 @@ def remove_directory(self, directory: Path): def find_npm(self) -> str: npm = shutil.which("npm") or shutil.which("npm.cmd") if not npm: - print("Could not find 'npm' on PATH. Make sure Node.js is installed.") + LOGGER.error("Could not find 'npm' on PATH. Make sure Node.js is installed.") sys.exit(1) return npm @@ -349,28 +349,28 @@ def verdaccio_login(self, registry: str, username: str, password: str) -> str | } payload = {"name": username, "password": password} - print(f"Logging in to Verdaccio at: {url}") + LOGGER.info(f"Logging in to Verdaccio at: {url}") resp = requests.put(url, headers=headers, data=json.dumps(payload), timeout=15) if not (200 <= resp.status_code < 300): - print(f"Login failed: {resp.status_code}") - print(resp.text) + LOGGER.error(f"Login failed: {resp.status_code}") + LOGGER.error(resp.text) return None try: data = resp.json() except ValueError: - print("Login response was not valid JSON:") - print(resp.text) + LOGGER.error("Login response was not valid JSON:") + LOGGER.error(resp.text) return None token = data.get("token") if not token: - print("Login succeeded but 'token' not found in response:") - print(data) + LOGGER.error("Login succeeded but 'token' not found in response:") + LOGGER.error(data) return None - print("Login succeeded, got token from Verdaccio.") + LOGGER.info("Login succeeded, got token from Verdaccio.") return token def npm_set_auth(self, registry: str, token: str, userconfig: str | None = None) -> int: @@ -389,15 +389,14 @@ def npm_set_auth(self, registry: str, token: str, userconfig: str | None = None) if userconfig: env["NPM_CONFIG_USERCONFIG"] = userconfig - print("\nSetting npm auth token with:") - print(" ", " ".join(cmd)) + LOGGER.info("Setting npm auth token with: %s", " ".join(cmd)) result = subprocess.run(cmd, env=env, capture_output=True, text=True) if result.stdout: - print(result.stdout) + LOGGER.info(result.stdout) if result.stderr: - print(result.stderr, file=sys.stderr) - print(f"npm config set exit code: {result.returncode}") + LOGGER.error(result.stderr) + LOGGER.info(f"npm config set exit code: {result.returncode}") return result.returncode def ensure_directories_exist(self, directories: list[Path]): diff --git a/classes/commands/RunTestsCommand.py b/classes/commands/RunTestsCommand.py index 11722d15..212e3d8d 100644 --- a/classes/commands/RunTestsCommand.py +++ b/classes/commands/RunTestsCommand.py @@ -150,7 +150,7 @@ async def _install_and_prepare_project_tool(self) -> Path: with open(tarball_filename, "wb") as f: shutil.copyfileobj(tarball_response.raw, f) - print(f"[INFO] Downloaded ProjectTool tarball: {tarball_filename}") + logging.info(f"Downloaded ProjectTool tarball: {tarball_filename}") import tarfile with tarfile.open(tarball_filename) as tf: diff --git a/classes/server/RemoteControlServer.py b/classes/server/RemoteControlServer.py index be5b79d3..d13df71b 100644 --- a/classes/server/RemoteControlServer.py +++ b/classes/server/RemoteControlServer.py @@ -308,7 +308,7 @@ async def _handle_manual_mode(self, reader: asyncio.StreamReader, writer: asynci if not response: return - print(f'> {response}') + LOGGER.info(f'> {response}') # Transition to FINISHED state self.state = State.FINISHED diff --git a/get_output_file_url.py b/get_output_file_url.py index 9033c1e8..35e23751 100644 --- a/get_output_file_url.py +++ b/get_output_file_url.py @@ -1,6 +1,10 @@ +import logging import os, json, requests, time import argparse +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +LOGGER = logging.getLogger(__name__) + # Define the directory path and file path dir_path = "" slack_file_path = os.path.join(dir_path, "slack_stats.json") @@ -26,7 +30,7 @@ # Check if path exists (could be a file or directory) if os.path.exists(output_file_path): - print("TF Output file located.") + LOGGER.info("TF Output file located.") # keep waiting until the output file artifact is ready def wait_for_artifact_ready(artifact_id, token, timeout=60, interval=5): @@ -56,13 +60,13 @@ def wait_for_artifact_ready(artifact_id, token, timeout=60, interval=5): # Artifact not found yet – keep trying pass else: - print(f"Unexpected error: {response.status_code}") + LOGGER.error(f"Unexpected error: {response.status_code}") return False - print("Waiting for artifact to be ready...") + LOGGER.info("Waiting for artifact to be ready...") time.sleep(interval) - print("Timeout: Artifact not ready in time.") + LOGGER.warning("Timeout: Artifact not ready in time.") return False @@ -86,7 +90,7 @@ def wait_for_artifact_ready(artifact_id, token, timeout=60, interval=5): # Update the value for field in data["attachments"][0]["fields"]: if field["title"] == "Output file": - print(f"Output file URL: {new_output_link}") + LOGGER.info(f"Output file URL: {new_output_link}") field["value"] = new_output_link break @@ -94,12 +98,12 @@ def wait_for_artifact_ready(artifact_id, token, timeout=60, interval=5): with open(slack_file_path, "w") as file: json.dump(data, file, indent=4) - print("Slack Stats JSON updated successfully.") + LOGGER.info("Slack Stats JSON updated successfully.") except Exception as e: - print({"error": str(e)}) + LOGGER.error({"error": str(e)}) else: - print("TF Output file does not exist.") + LOGGER.warning("TF Output file does not exist.") diff --git a/tf_compare.py b/tf_compare.py index ca961d87..d7f39ab3 100644 --- a/tf_compare.py +++ b/tf_compare.py @@ -1,12 +1,16 @@ +import logging import requests, zipfile, sys, json, os, glob, re, shutil, time, fnmatch from dotenv import load_dotenv + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +LOGGER = logging.getLogger(__name__) # Load the .env file load_dotenv() # Access the variables baseSaveLocation = os.getenv("BASE_SAVE_LOCATION") -print("Base Save Location:", baseSaveLocation) +LOGGER.info("Base Save Location: %s", baseSaveLocation) from pathlib import Path from datetime import datetime, timezone, timedelta @@ -101,9 +105,9 @@ def get_workflow_runs(): if len(_artifactRunID) >= 1: get_artifact_URL() else: - print("Valid workflow not used, only Beta, Monthly or Red on the develop branch is accepted for the TF Compare script") + LOGGER.error("Valid workflow not used, only Beta, Monthly or Red on the develop branch is accepted for the TF Compare script") else: - print(f"Failed to get workflow runs. HTTP Status: {response.status_code}") + LOGGER.error(f"Failed to get workflow runs. HTTP Status: {response.status_code}") @@ -140,20 +144,20 @@ def get_artifact_URL(): _artifactID.append(artifact['id']) _download_artifacts_url[runState] = artifact.get("archive_download_url") else: - print(f"Failed to artifact URL. HTTP Status: {response.status_code}") + LOGGER.error(f"Failed to artifact URL. HTTP Status: {response.status_code}") # Time to download the artifact files, ensure the current run has a valid artifact file if (len(_download_artifacts_url) > 0) and _download_artifacts_url.get('Current'): download_github_artifact(_download_artifacts_url) else: - print(f"No artifact files available in the current workflow run!\nTF Compare script will not continue") + LOGGER.error(f"No artifact files available in the current workflow run!\nTF Compare script will not continue") def download_github_artifact(_download_artifacts_url): # iterate through the _download_artifacts_url array and download each artifact file urlCount = 0 - print("Downloading artifact files") + LOGGER.info("Downloading artifact files") for url in _download_artifacts_url: @@ -176,11 +180,11 @@ def download_github_artifact(_download_artifacts_url): urlCount +=1 else: - print(f"Failed to download artifact. HTTP Status: {response.status_code}") + LOGGER.error(f"Failed to download artifact. HTTP Status: {response.status_code}") # time to compare the artifact files if len(artifact_files) >= 1: - print("Artifacts successfully downloaded") + LOGGER.info("Artifacts successfully downloaded") compare_artifacts(artifact_files) @@ -206,7 +210,7 @@ def unzip_artifact_files(save_path, zipfilename):# # Extract each josn file zip_ref.extract(art_file, extract_to) else: - print("No files found in the artifact archive.") # Print error details + LOGGER.warning("No files found in the artifact archive.") return artifact_files @@ -224,7 +228,7 @@ def compare_artifacts(artifact_files): allTestFiles = {} fileCount = 1 - print("Processing and Comparing artifact data") + LOGGER.info("Processing and Comparing artifact data") # for each data directory for data in saveLocation: @@ -243,7 +247,7 @@ def compare_artifacts(artifact_files): # xUnit_windows_VM_2, xUnit_windows_YYC_2 - Previous Test Run allTestFiles[f"{art_file}_{fileCount}"] = json.load(file) else: - print(f"File in {file_path} does not exist.") + LOGGER.warning(f"File in {file_path} does not exist.") fileCount +=1 if len(allTestFiles) > 0: @@ -416,7 +420,7 @@ def compare_artifacts(artifact_files): testCounter +=1 elif len(failsWrapper[0]) + len(failsWrapper[1]) + len(failsWrapper[2]) == 0: - print(f"\nNo fails have been identified in this run for {compiler[cIndex]}.") # Print error details + LOGGER.info(f"No fails have been identified in this run for {compiler[cIndex]}.") file.write("\n************************************** NEW FAILS ***************************************\n") @@ -451,7 +455,7 @@ def compare_artifacts(artifact_files): file.write("****************************************************************************************\n") # confirm successful creation of output file - print("\nTEXT file 'TF_Output.txt' was created successfully!") + LOGGER.info("TEXT file 'TF_Output.txt' was created successfully!") # Remove all downloaded artifacts files @@ -465,8 +469,8 @@ def compare_artifacts(artifact_files): if os.path.isfile(file): # Ensure it's a file (not a folder) os.remove(file) - print("\nArtifact comparison has completed") - print("All downloaded artifact files deleted.") + LOGGER.info("Artifact comparison has completed") + LOGGER.info("All downloaded artifact files deleted.") # _artifactRunID # _artifactID @@ -484,7 +488,7 @@ def compare_artifacts(artifact_files): # build JSON file content for Slack Notification - print("\nCreating Slack JSON Stats file") + LOGGER.info("Creating Slack JSON Stats file") slack_stats["text"] = f"*{RTVersion} {workflow.split(".")[0]} Test Results Summary*" slack_stats["Runtime-Version"] = RTVersion slack_stats["attachments"] = [ @@ -519,7 +523,7 @@ def compare_artifacts(artifact_files): with open("slack_stats.json", "w") as slackfile: # Convert the list to a JSON-formatted string json.dump(slack_stats, slackfile, indent=4) - print("\nJSON file 'slack_stats.json' was created successfully!") + LOGGER.info("JSON file 'slack_stats.json' was created successfully!") #Get failed test code block and lines @@ -570,9 +574,9 @@ def get_code(testname, testsuite): return [function_code, permalink] else: - print("Function not found in file.") + LOGGER.warning("Function not found in file.") else: - print(f"Failed to fetch file. HTTP Status: {response.status_code}") + LOGGER.error(f"Failed to fetch file. HTTP Status: {response.status_code}") # create new bug report / comment on existing report @@ -591,12 +595,12 @@ def log_fail(testName, failDetails, compiler, test_code_details): headers['Accept'] = 'application/vnd.github.v3+json' # Search all repositories - print(f"\nSearching for test: {testName}") + LOGGER.info(f"Searching for test: {testName}") issue_search = next((data for repo in repos if (data := get_issues(repo, testName, compiler))), None) if issue_search != None: - print(f"Report has been found for: {testName} in Repo: {issue_search[1]}") + LOGGER.info(f"Report has been found for: {testName} in Repo: {issue_search[1]}") # fail has already been written up # check its state (open/closed) for report in issue_search[0]['items']: @@ -609,9 +613,9 @@ def log_fail(testName, failDetails, compiler, test_code_details): response = requests.patch(report['url'], headers=headers, json=issue_data) if response.status_code == 200: - print("This issue is currently marked as closed!") - print(f"Issue: {report['number']} - {testName}, successfully reopened") - print(f"Bug Report URL{report['html_url']}") + LOGGER.info("This issue is currently marked as closed!") + LOGGER.info(f"Issue: {report['number']} - {testName}, successfully reopened") + LOGGER.info(f"Bug Report URL: {report['html_url']}") # add 1 to the reopened count total_reopened_reports += 1 # add new comment to bug report @@ -633,15 +637,13 @@ def log_fail(testName, failDetails, compiler, test_code_details): response = requests.post(report['comments_url'], headers=headers, json=comment_data) if response.status_code == 201: - print(f"Issue: {report['number']} - {testName}, new comment successfully added to report") - # print(f"Bug Report URL: {report['html_url']}") + LOGGER.info(f"Issue: {report['number']} - {testName}, new comment successfully added to report") break else: - print(f"Issue: {report['number']} - {testName}, adding a new comment was unsuccessful!") - # print(f"Bug Report URL: {report['html_url']}") + LOGGER.warning(f"Issue: {report['number']} - {testName}, adding a new comment was unsuccessful!") break else: - print(f"Issue: {report['number']} - {testName}, could not be reopened") + LOGGER.warning(f"Issue: {report['number']} - {testName}, could not be reopened") # the found report is still open and unresolved elif report['state'] == 'open': @@ -689,23 +691,21 @@ def log_fail(testName, failDetails, compiler, test_code_details): response = requests.post(report['comments_url'], headers=headers, json=comment_data) if response.status_code == 201: - print(f"Issue: {report['number']} - {testName}, new comment successfully added to report") - # print(f"Bug Report URL: {report['html_url']}") + LOGGER.info(f"Issue: {report['number']} - {testName}, new comment successfully added to report") break else: - print(f"Issue: {report['number']} - {testName}, adding a new comment was unsuccessful!") - # print(f"Bug Report URL: {report['html_url']}") + LOGGER.warning(f"Issue: {report['number']} - {testName}, adding a new comment was unsuccessful!") break else: - print(f"Error: {response.status_code} - {response.json()}") + LOGGER.error(f"Error: {response.status_code} - {response.json()}") # return existing bug report url - print(f"Bug Report URL: {report['html_url']}") + LOGGER.info(f"Bug Report URL: {report['html_url']}") return f"{report['html_url']}" else: # new report to be written up # Look at adding the new reports to a new holding repo - print(f"No report found for: {testName}") + LOGGER.info(f"No report found for: {testName}") if failDetails['errorType'] == 'error': if 'description' in failDetails['errorDetails']: @@ -784,8 +784,8 @@ def log_fail(testName, failDetails, compiler, test_code_details): if response.status_code == 201: issue_data = response.json() - print(f"Issue: {issue_data['number']} - {testName}, successfully created!") - print(f"Bug Report URL: {issue_data['url']}") + LOGGER.info(f"Issue: {issue_data['number']} - {testName}, successfully created!") + LOGGER.info(f"Bug Report URL: {issue_data['url']}") # add 1 to the new report created count total_new_reports += 1 diff --git a/utils/async_utils.py b/utils/async_utils.py index 62ffdf7b..fbba7324 100644 --- a/utils/async_utils.py +++ b/utils/async_utils.py @@ -140,7 +140,7 @@ async def check_keypress_unix(): break await loop.run_in_executor(None, sys.stdin.read, 1) if sys.stdin.read(1) == ' ': - print("Space key pressed. Stopping server.") + LOGGER.info("Space key pressed. Stopping server.") if stop_event: stop_event.set() # Signal to stop the server break @@ -154,16 +154,16 @@ async def check_keypress_win(): break await asyncio.sleep(0.1) if msvcrt.kbhit() and msvcrt.getch() == b' ': - print("Space key pressed. Stopping server.") + LOGGER.info("Space key pressed. Stopping server.") if stop_event: stop_event.set() # Signal to stop the server break if "CI" in os.environ: - print("CI environment detected. Skipping wait for space key.") + LOGGER.info("CI environment detected. Skipping wait for space key.") return - print("Press the space key to stop the server...") + LOGGER.info("Press the space key to stop the server...") if sys.platform == 'win32': await check_keypress_win() else: