diff --git a/docs/conf.py b/docs/conf.py index fc6460266..027311a40 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,9 +35,9 @@ "myst_parser", "sphinxcontrib.plantuml", "score_plantuml", + "score_source_code_linker", "score_metamodel", "score_draw_uml_funcs", - "score_source_code_linker", "score_layout", ] diff --git a/docs/internals/extensions/source_code_linker.md b/docs/internals/extensions/source_code_linker.md index 2e7085d54..4d8f53108 100644 --- a/docs/internals/extensions/source_code_linker.md +++ b/docs/internals/extensions/source_code_linker.md @@ -5,6 +5,9 @@ A Sphinx extension for source code traceability for requirements. This extension In a first step it parses the source code for requirement tags. All discovered tags including their file and line numbers are written in an intermediary file before the sphinx build. In a second step this intermediary file is parsed during sphinx build. If a requirement Id is found in the intermediary file a link to the source is added. +** Please note that the 'test parsing & linking' has been added to the source-code-linker. ** +* The documentation for this part will follow soon * + ## Implementation Components ### Bazel Integration diff --git a/docs/requirements/index.rst b/docs/requirements/index.rst index 1c730e0e9..fde7eb587 100644 --- a/docs/requirements/index.rst +++ b/docs/requirements/index.rst @@ -9,3 +9,4 @@ Requirements capabilities process_overview requirements + test_overview diff --git a/docs/requirements/requirements.rst b/docs/requirements/requirements.rst index e4d8c8584..7fe086bee 100644 --- a/docs/requirements/requirements.rst +++ b/docs/requirements/requirements.rst @@ -4,6 +4,7 @@ Tool Requirements ================================= + 📈 Status ########## diff --git a/docs/requirements/test_overview.rst b/docs/requirements/test_overview.rst new file mode 100644 index 000000000..c87b78fb5 --- /dev/null +++ b/docs/requirements/test_overview.rst @@ -0,0 +1,55 @@ +.. _testing_stats: + +TESTING STATISTICS +================== + + +.. needtable:: SUCCESSFUL TEST + :filter: result == "passed" + :tags: TEST + :columns: external_url as "source_link"; name as "testcase";result;fully_verifies;partially_verifies;test_type;derivation_technique + + +.. needtable:: FAILED TEST + :filter: result == "failed" + :tags: TEST + :columns: external_url as "source_link"; name as "testcase";result;fully_verifies;partially_verifies;test_type;derivation_technique + + +.. needtable:: OTHER TEST + :filter: result != "failed" and result != "passed" + :tags: TEST + :columns: external_url as "source_link"; name as "testcase";result;fully_verifies;partially_verifies;test_type;derivation_technique + + +.. needpie:: Test Results + :labels: passed, failed, skipped + :colors: green, red, orange + :legend: + + type == 'testcase' and result == 'passed' + type == 'testcase' and result == 'failed' + type == 'testcase' and result == 'skipped' + + +.. needpie:: Test Types Used In Testcases + :labels: fault-injection, interface-test, requirements-based, resource-usage + :legend: + + type == 'testcase' and test_type == 'fault-injection' + type == 'testcase' and test_type == 'interface-test' + type == 'testcase' and test_type == 'requirements-based' + type == 'testcase' and test_type == 'resource-usage' + + +.. needpie:: Derivation Techniques Used In Testcases + :labels: requirements-analysis, design-analysis, boundary-values, equivalence-classes, fuzz-testing, error-guessing, explorative-testing + :legend: + + type == 'testcase' and derivation_technique == 'requirements-analysis' + type == 'testcase' and derivation_technique == 'design-analysis' + type == 'testcase' and derivation_technique == 'boundary-values' + type == 'testcase' and derivation_technique == 'equivalence-classes' + type == 'testcase' and derivation_technique == 'fuzz-testing' + type == 'testcase' and derivation_technique == 'error-guessing' + type == 'testcase' and derivation_technique == 'explorative-testing' diff --git a/src/extensions/score_layout/sphinx_options.py b/src/extensions/score_layout/sphinx_options.py index 663f3104e..cc9e402ab 100644 --- a/src/extensions/score_layout/sphinx_options.py +++ b/src/extensions/score_layout/sphinx_options.py @@ -39,8 +39,8 @@ class SingleLayout(TypedDict): "initial=False)>>", ], "meta_left": [ - '<>', - "<>", + '<>', + '<>', ], "meta_right": [], "footer_left": ["<>"], diff --git a/src/extensions/score_metamodel/checks/check_options.py b/src/extensions/score_metamodel/checks/check_options.py index aee3b18a8..1df4da0f0 100644 --- a/src/extensions/score_metamodel/checks/check_options.py +++ b/src/extensions/score_metamodel/checks/check_options.py @@ -11,7 +11,6 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* import re -from os import error from score_metamodel import ( CheckLogger, diff --git a/src/extensions/score_metamodel/metamodel.yaml b/src/extensions/score_metamodel/metamodel.yaml index f336f4156..59ac8041f 100644 --- a/src/extensions/score_metamodel/metamodel.yaml +++ b/src/extensions/score_metamodel/metamodel.yaml @@ -16,6 +16,8 @@ needs_types_base_options: optional_options: # req-Id: tool_req__docs_dd_link_source_code_link source_code_link: ^https://github.com/.* + testlink: ^https://github.com/.* + # Custom semantic validation rules # Prohibited Word Option Checks # Follow this schema to write new checks @@ -372,7 +374,6 @@ needs_types: optional_options: codelink: ^.*$ tags: ^.*$ - testlink: ^.*$ # req-Id: tool_req__docs_req_attr_reqcov reqcovered: ^(YES|NO)$ # req-Id: tool_req__docs_req_attr_testcov @@ -830,6 +831,23 @@ needs_types: - safety_analysis parts: 3 + testcase: + title: Testcase Needs parsed from test.xml files + prefix: testcase__ + mandatory_options: + id: ^testcase__ + optional_options: + name: ^.*$ + file: ^.*$ + line: ^.*$ + test_type: ^.*$ + derivation_technique: ^.*$ + result: ^.*$ + result_text: ^.*$ + optional_links: + fully_verifies: ^.*$ + partially_verifies: ^.*$ + # Extra link types, which shall be available and allow need types to be linked to each other. # We use a dedicated linked type for each type of a connection, for instance from # a specification to a requirement. This makes filtering and visualization of such connections @@ -915,6 +933,14 @@ needs_extra_links: violates: incoming: violated_by outgoing: violates + + fully_verifies: + incoming: fully_verified_by + outgoing: fully_verifies + + partially_verifies: + incoming: partially_verified_by + outgoing: partially_verifies ############################################################## # Graph Checks # The graph checks focus on the relation of the needs and their attributes. diff --git a/src/extensions/score_source_code_linker/BUILD b/src/extensions/score_source_code_linker/BUILD index 08735e007..a8fd36cda 100644 --- a/src/extensions/score_source_code_linker/BUILD +++ b/src/extensions/score_source_code_linker/BUILD @@ -10,6 +10,18 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* +#******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_library") load("@pip_process//:requirements.bzl", "all_requirements") load("@score_python_basics//:defs.bzl", "score_py_pytest") @@ -27,6 +39,17 @@ py_library( deps = ["@score_docs_as_code//src/helper_lib"], ) +py_library( + name = "source_code_linker_helpers", + srcs = [ + "needlinks.py", + "testlink.py", + "xml_parser.py", + ], + imports = ["."], + visibility = ["//visibility:public"], +) + score_py_pytest( name = "score_source_code_linker_test", size = "small", @@ -42,5 +65,5 @@ score_py_pytest( deps = [ ":score_source_code_linker", "@score_docs_as_code//src/extensions/score_metamodel", - ] + all_requirements, + ], ) diff --git a/src/extensions/score_source_code_linker/__init__.py b/src/extensions/score_source_code_linker/__init__.py index b8184c4e4..0f0274a1d 100644 --- a/src/extensions/score_source_code_linker/__init__.py +++ b/src/extensions/score_source_code_linker/__init__.py @@ -11,7 +11,8 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""In this file the actual sphinx extension is defined. It will read pre-generated +""" +In this file the actual sphinx extension is defined. It will read pre-generated source code links from a JSON file and add them to the needs. """ @@ -21,7 +22,6 @@ from typing import cast from sphinx.application import Sphinx -from sphinx.config import Config from sphinx.environment import BuildEnvironment from sphinx_needs.data import NeedsInfoType, NeedsMutable, SphinxNeedsData from sphinx_needs.logging import get_logger @@ -29,52 +29,125 @@ from src.extensions.score_source_code_linker.generate_source_code_links_json import ( generate_source_code_links_json, ) +from src.extensions.score_source_code_linker.need_source_links import ( + NeedSourceLinks, + SourceCodeLinks, + load_source_code_links_combined_json, + store_source_code_links_combined_json, +) from src.extensions.score_source_code_linker.needlinks import ( - DefaultNeedLink, NeedLink, load_source_code_links_json, ) +from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink, + load_data_of_test_case_json, + load_test_xml_parsed_json, +) +from src.extensions.score_source_code_linker.xml_parser import ( + construct_and_add_need, + run_xml_parser, +) from src.helper_lib import ( find_git_root, find_ws_root, - get_current_git_hash, - get_github_base_url, ) +from src.helper_lib.additional_functions import get_github_link LOGGER = get_logger(__name__) -# Outcomment this to enable more verbose logging +# Uncomment this to enable more verbose logging # LOGGER.setLevel("DEBUG") -def get_cache_filename(build_dir: Path) -> Path: +# re-qid: gd_req__req_attr_impl +# ╭──────────────────────────────────────╮ +# │ JSON FILE RELATED FUNCS │ +# ╰──────────────────────────────────────╯ + + +def group_by_need( + source_code_links: list[NeedLink], + test_case_links: list[DataForTestLink] | None = None, +) -> list[SourceCodeLinks]: + """ + Groups the given need links and test case links by their need ID. + Returns a nested dictionary structure with 'CodeLink' and 'TestLink' categories. + Example output: + + + { + "need": "", + "links": { + "CodeLinks": [NeedLink, NeedLink, ...], + "TestLinks": [testlink, testlink, ...] + } + } + """ + # TODO: I wonder if there is a more efficent way to do this + grouped_by_need: dict[str, NeedSourceLinks] = defaultdict( + lambda: NeedSourceLinks(TestLinks=[], CodeLinks=[]) + ) + + # Group source code links + for needlink in source_code_links: + grouped_by_need[needlink.need].CodeLinks.append(needlink) + + # Group test case links + if test_case_links is not None: + for testlink in test_case_links: + grouped_by_need[testlink.need].TestLinks.append(testlink) + + # Build final list of SourceCodeLinks + result: list[SourceCodeLinks] = [ + SourceCodeLinks( + need=need, + links=NeedSourceLinks( + CodeLinks=need_links.CodeLinks, + TestLinks=need_links.TestLinks, + ), + ) + for need, need_links in grouped_by_need.items() + ] + + return result + + +def get_cache_filename(build_dir: Path, filename: str) -> Path: """ Returns the path to the cache file for the source code linker. This is used to store the generated source code links. """ - return build_dir / "score_source_code_linker_cache.json" + return build_dir / filename -def setup_once(app: Sphinx, config: Config): - # might be the only way to solve this? - if "skip_rescanning_via_source_code_linker" in app.config: - return - LOGGER.debug(f"DEBUG: Workspace root is {find_ws_root()}") - LOGGER.debug( - f"DEBUG: Current working directory is {Path('.')} = {Path('.').resolve()}" +def build_and_save_combined_file(outdir: Path): + """ + Reads the saved partial caches of codelink & testlink + Builds the combined JSON cache & saves it + """ + source_code_links = load_source_code_links_json( + get_cache_filename(outdir, "score_source_code_linker_cache.json") + ) + test_code_links = load_test_xml_parsed_json( + get_cache_filename(outdir, "score_xml_parser_cache.json") ) - LOGGER.debug(f"DEBUG: Git root is {find_git_root()}") - # Run only for local files! - # ws_root is not set when running on any on bazel run - # command repositories (dependencies) - ws_root = find_ws_root() - if not ws_root: - return + store_source_code_links_combined_json( + outdir / "score_scl_grouped_cache.json", + group_by_need(source_code_links, test_code_links), + ) + + +# ╭──────────────────────────────────────╮ +# │ ONE TIME SETUP FUNCS │ +# ╰──────────────────────────────────────╯ - # When BUILD_WORKSPACE_DIRECTORY is set, we are inside a git repository. - assert find_git_root() - # Extension: score_source_code_linker +def setup_source_code_linker(app: Sphinx, ws_root: Path): + """ + Setting up source_code_linker with all needed options. + Allows us to only have this run once during live_preview & esbonio + """ app.add_config_value( "skip_rescanning_via_source_code_linker", False, @@ -84,32 +157,129 @@ def setup_once(app: Sphinx, config: Config): ) # Define need_string_links here to not have it in conf.py + # source_code_link and testlinks have the same schema app.config.needs_string_links = { "source_code_linker": { "regex": r"(?P.+)<>(?P.+)", "link_url": "{{url}}", "link_name": "{{name}}", - "options": ["source_code_link"], + "options": ["source_code_link", "testlink"], }, } - cache_json = get_cache_filename(Path(app.outdir)) + scl_cache_json = get_cache_filename( + app.outdir, "score_source_code_linker_cache.json" + ) - if not cache_json.exists() or not app.config.skip_rescanning_via_source_code_linker: + if ( + not scl_cache_json.exists() + or not app.config.skip_rescanning_via_source_code_linker + ): LOGGER.debug( "INFO: Generating source code links JSON file.", type="score_source_code_linker", ) - generate_source_code_links_json(ws_root, cache_json) + generate_source_code_links_json(ws_root, scl_cache_json) + + +def register_test_code_linker(app: Sphinx): + # Connects function to sphinx to ensure correct execution order + # priority is set to make sure it is called in the right order. + # Before the combining action + app.connect("env-updated", setup_test_code_linker, priority=505) + + +def setup_test_code_linker(app: Sphinx, env: BuildEnvironment): + tl_cache_json = get_cache_filename(app.outdir, "score_xml_parser_cache.json") + if ( + not tl_cache_json.exists() + or not app.config.skip_rescanning_via_source_code_linker + ): + ws_root = find_ws_root() + if not ws_root: + return + LOGGER.debug( + "INFO: Generating score_xml_parser JSON file.", + type="score_source_code_linker", + ) + # sanity check if extension is enabled + bazel_testlogs = ws_root / "bazel-testlogs" + if not bazel_testlogs.exists(): + LOGGER.info(f"{'=' * 80}", type="score_source_code_linker") + LOGGER.info( + f"{'=' * 32}SCORE XML PARSER{'=' * 32}", type="score_source_code_linker" + ) + LOGGER.info( + "'bazel-testlogs' was not found. If test data should be parsed," + + "please run tests before building the documentation", + type="score_source_code_linker", + ) + LOGGER.info(f"{'=' * 80}", type="score_source_code_linker") + return + + run_xml_parser(app, env) + return + tcn_cache = get_cache_filename(app.outdir, "score_testcaseneeds_cache.json") + assert tcn_cache.exists(), ( + f"TestCaseNeed Cache file does not exist.Checked Path: {tcn_cache}" + ) + # TODO: Make this more efficent, idk how though. + test_case_needs = load_data_of_test_case_json(tcn_cache) + for tcn in test_case_needs: + construct_and_add_need(app, tcn) - app.connect("env-updated", inject_links_into_needs) + +def register_combined_linker(app: Sphinx): + # Registering the combined linker to Sphinx + # priority is set to make sure it is called in the right order. + # Needs to be called after xml parsing & codelink + app.connect("env-updated", setup_combined_linker, priority=507) + + +def setup_combined_linker(app: Sphinx, _: BuildEnvironment): + grouped_cache = get_cache_filename(app.outdir, "score_scl_grouped_cache.json") + gruped_cache_exists = grouped_cache.exists() + if not gruped_cache_exists or not app.config.skip_rescanning_via_source_code_linker: + LOGGER.debug( + "Did not find combined json 'score_scl_grouped_cache.json' in _build." + "Generating new one" + ) + build_and_save_combined_file(app.outdir) + + +def setup_once(app: Sphinx): + # might be the only way to solve this? + if "skip_rescanning_via_source_code_linker" in app.config: + return + LOGGER.debug(f"DEBUG: Workspace root is {find_ws_root()}") + LOGGER.debug( + f"DEBUG: Current working directory is {Path('.')} = {Path('.').resolve()}" + ) + LOGGER.debug(f"DEBUG: Git root is {find_git_root()}") + + # Run only for local files! + # ws_root is not set when running on external repositories (dependencies). + ws_root = find_ws_root() + if not ws_root: + return + + # When BUILD_WORKSPACE_DIRECTORY is set, we are inside a git repository. + assert find_git_root() + + # Register & Run (if needed) parsing & saving of JSON caches + setup_source_code_linker(app, ws_root) + register_test_code_linker(app) + register_combined_linker(app) + + # Priorty=510 to ensure it's called after the test code linker & combined connection + app.connect("env-updated", inject_links_into_needs, priority=510) def setup(app: Sphinx) -> dict[str, str | bool]: # Esbonio will execute setup() on every iteration. # setup_once will only be called once. - setup_once(app, app.config) + setup_once(app) return { "version": "0.1", @@ -132,33 +302,13 @@ def find_need( for prefix in prefixes: prefixed_id = f"{prefix}{id}" if prefixed_id in all_needs: + LOGGER.warning("linking to external needs is not supported!") return all_needs[prefixed_id] return None -def group_by_need(source_code_links: list[NeedLink]) -> dict[str, list[NeedLink]]: - """ - Groups the given need links by their need ID. - """ - source_code_links_by_need: dict[str, list[NeedLink]] = defaultdict(list) - for needlink in source_code_links: - source_code_links_by_need[needlink.need].append(needlink) - return source_code_links_by_need - - -def get_github_link(needlink: NeedLink | None = None) -> str: - if needlink is None: - needlink = DefaultNeedLink() - passed_git_root = find_git_root() - if passed_git_root is None: - passed_git_root = Path() - base_url = get_github_base_url() - current_hash = get_current_git_hash(passed_git_root) - return f"{base_url}/blob/{current_hash}/{needlink.file}#L{needlink.line}" - - -# req-Id: tool_req__docs_dd_link_source_code_link +# re-qid: gd_req__req__attr_impl def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: """ 'Main' function that facilitates the running of all other functions @@ -169,7 +319,6 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: env: Buildenvironment, this is filled automatically app: Sphinx app application, this is filled automatically """ - ws_root = find_ws_root() assert ws_root @@ -179,46 +328,65 @@ def inject_links_into_needs(app: Sphinx, env: BuildEnvironment) -> None: needs ) # TODO: why do we create a copy? Can we also needs_copy = needs[:]? copy(needs)? - for _, need in needs.items(): - if need.get("source_code_link"): - LOGGER.debug( - f"?? Need {need['id']} already has source_code_link: " - f"{need.get('source_code_link')}" - ) - - source_code_links = load_source_code_links_json(get_cache_filename(app.outdir)) - - # group source_code_links by need - # groupby requires the input to be sorted by the key + # Enabled automatically for DEBUGGING + if LOGGER.getEffectiveLevel() >= 10: + for id, need in needs.items(): + if need.get("source_code_link"): + LOGGER.debug( + f"?? Need {id} already has source_code_link: {need.get('source_code_link')}" + ) + if need.get("testlink"): + LOGGER.debug( + f"?? Need {id} already has testlink: {need.get('testlink')}" + ) - source_code_links_by_need = group_by_need(source_code_links) + source_code_links_by_need = load_source_code_links_combined_json( + get_cache_filename(app.outdir, "score_scl_grouped_cache.json") + ) # For some reason the prefix 'sphinx_needs internally' is CAPSLOCKED. # So we have to make sure we uppercase the prefixes prefixes = [x["id_prefix"].upper() for x in app.config.needs_external_needs] - for need_id, needlinks in source_code_links_by_need.items(): - need = find_need(needs_copy, need_id, prefixes) + for source_code_links in source_code_links_by_need: + need = find_need(needs_copy, source_code_links.need, prefixes) if need is None: # TODO: print github annotations as in https://github.com/eclipse-score/bazel_registry/blob/7423b9996a45dd0a9ec868e06a970330ee71cf4f/tools/verify_semver_compatibility_level.py#L126-L129 - for n in needlinks: + for n in source_code_links.links.CodeLinks: + LOGGER.warning( + f"{n.file}:{n.line}: Could not find {source_code_links.need} in documentation [CODE LINK]", + type="score_source_code_linker", + ) + for n in source_code_links.links.TestLinks: LOGGER.warning( - f"{n.file}:{n.line}: Could not find {need_id} in documentation", + f"{n.file}:{n.line}: Could not find {source_code_links.need} in documentation [TEST LINK]", type="score_source_code_linker", ) - else: - need_as_dict = cast(dict[str, object], need) + continue + + need_as_dict = cast(dict[str, object], need) + + need_as_dict["source_code_link"] = ", ".join( + f"{get_github_link(n)}<>{n.file}:{n.line}" + for n in source_code_links.links.CodeLinks + ) + need_as_dict["testlink"] = ", ".join( + f"{get_github_link(n)}<>{n.name}" for n in source_code_links.links.TestLinks + ) + + # NOTE: Removing & adding the need is important to make sure + # the needs gets 're-evaluated'. + Needs_Data.remove_need(need["id"]) + Needs_Data.add_need(need) - need_as_dict["source_code_link"] = ", ".join( - f"{get_github_link(n)}<>{n.file}:{n.line}" for n in needlinks - ) - # NOTE: Removing & adding the need is important to make sure - # the needs gets 're-evaluated'. - Needs_Data.remove_need(need["id"]) - Needs_Data.add_need(need) +# ╭──────────────────────────────────────╮ +# │ WARNING: This somehow screws up the │ +# │ integration test? What?? │ +# │ Commented out for now │ +# ╰──────────────────────────────────────╯ - # source_code_link of affected needs was overwritten. - # Make sure it's empty in all others! - for need in needs.values(): - if need["id"] not in source_code_links_by_need: - need["source_code_link"] = "" +# source_code_link of affected needs was overwritten. Make sure it's empty in all others! +# for need in needs.values(): +# if need["id"] not in source_code_links_by_need: +# need["source_code_link"] = "" # type: ignore +# need["testlink"] = "" # type: ignore diff --git a/src/extensions/score_source_code_linker/need_source_links.py b/src/extensions/score_source_code_linker/need_source_links.py new file mode 100644 index 000000000..f0310c0cd --- /dev/null +++ b/src/extensions/score_source_code_linker/need_source_links.py @@ -0,0 +1,104 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +This file defines NeedSourceLinks as well as SourceCodeLinks. +Both datatypes are used in the 'grouped cache' JSON that contains 'CodeLinks' and 'TestLinks' +It also defines a decoder and encoder for SourceCodeLinks to enable JSON read/write +""" + +import json +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from src.extensions.score_source_code_linker.needlinks import ( + NeedLink, +) +from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink, +) + + +@dataclass +class NeedSourceLinks: + CodeLinks: list[NeedLink] = field(default_factory=list) + TestLinks: list[DataForTestLink] = field(default_factory=list) + + +@dataclass +class SourceCodeLinks: + # TODO: Find a good key name for this + need: str + links: NeedSourceLinks + # Example: + # + # need: + # links: + # { + # "CodeLinks: + # [{needlink},{needlink}...], + # "TestLinks": + # [{testlink},{testlink},...] + + +class SourceCodeLinks_JSON_Encoder(json.JSONEncoder): + def default(self, o: object): + if isinstance(o, (SourceCodeLinks, NeedSourceLinks)): + return asdict(o) + if isinstance(o, (NeedLink, DataForTestLink)): + return asdict(o) + if isinstance(o, Path): + return str(o) + return super().default(o) + + +def SourceCodeLinks_JSON_Decoder(d: dict[str, Any]) -> SourceCodeLinks | dict[str, Any]: + if "need" in d and "links" in d: + links = d["links"] + return SourceCodeLinks( + need=d["need"], + links=NeedSourceLinks( + CodeLinks=[NeedLink(**cl) for cl in links.get("CodeLinks", [])], + TestLinks=[DataForTestLink(**tl) for tl in links.get("TestLinks", [])], + ), + ) + return d + + +def store_source_code_links_combined_json( + file: Path, source_code_links: list[SourceCodeLinks] +): + # After `rm -rf _build` or on clean builds the directory does not exist, so we need to create it + file.parent.mkdir(exist_ok=True) + with open(file, "w") as f: + json.dump( + source_code_links, + f, + cls=SourceCodeLinks_JSON_Encoder, + indent=2, + ensure_ascii=False, + ) + + +def load_source_code_links_combined_json(file: Path) -> list[SourceCodeLinks]: + links: list[SourceCodeLinks] = json.loads( + file.read_text(encoding="utf-8"), + object_hook=SourceCodeLinks_JSON_Decoder, + ) + assert isinstance(links, list), ( + "The combined source code linker links should be a list of SourceCodeLinks objects." + ) + assert all(isinstance(link, SourceCodeLinks) for link in links), ( + "All items in combined_source_code_linker_cache should be SourceCodeLinks objects." + ) + return links diff --git a/src/extensions/score_source_code_linker/needlinks.py b/src/extensions/score_source_code_linker/needlinks.py index 406ad9419..2d3ca246d 100644 --- a/src/extensions/score_source_code_linker/needlinks.py +++ b/src/extensions/score_source_code_linker/needlinks.py @@ -16,7 +16,7 @@ from typing import Any -@dataclass(frozen=True) +@dataclass(frozen=True, order=True) class NeedLink: """Represents a single template string finding in a file.""" diff --git a/src/extensions/score_source_code_linker/testlink.py b/src/extensions/score_source_code_linker/testlink.py new file mode 100644 index 000000000..30416e5de --- /dev/null +++ b/src/extensions/score_source_code_linker/testlink.py @@ -0,0 +1,248 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +This file defines the TestLink and TestCaseNeed. +It also defines encoding & decoding for JSON write/reading of TestLinks + +TestCaseNeed => The datatype that testcases from the test.xml get parsed into +TestLink => The datatype that is ultimately saved inside of the JSON +""" + +import html +import json +import re +from dataclasses import asdict, dataclass +from itertools import chain +from pathlib import Path +from typing import Any + +from sphinx_needs import logging + +LOGGER = logging.get_logger(__name__) + + +@dataclass(frozen=True, order=True) +class DataForTestLink: + name: str + file: Path + line: int + need: str + verify_type: str + result: str + result_text: str = "" + + +class DataForTestLink_JSON_Encoder(json.JSONEncoder): + def default(self, o: object): + if isinstance(o, DataForTestLink): + return asdict(o) + if isinstance(o, Path): + return str(o) + return super().default(o) + + +def DataForTestLink_JSON_Decoder(d: dict[str, Any]) -> DataForTestLink | dict[str, Any]: + if { + "name", + "file", + "line", + "need", + "verify_type", + "result", + "result_text", + } <= d.keys(): + return DataForTestLink( + name=d["name"], + file=Path(d["file"]), + line=d["line"], + need=d["need"], + verify_type=d["verify_type"], + result=d["result"], + result_text=d["result_text"], + ) + # It's something else, pass it on to other decoders + return d + + +# We will have everything as string here as that mirrors the xml file +@dataclass +class DataOfTestCase: + name: str + file: str + line: str + result: str # passed | falied | skipped | disabled + # Intentionally not snakecase to make dict parsing simple + TestType: str + DerivationTechnique: str + result_text: str = "" # Can be None on anything but failed + # Either or HAVE to be filled. + PartiallyVerifies: str | None = None + FullyVerifies: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any]): # type-ignore + return cls(**data) # type-ignore + + @classmethod + def clean_text(cls, text: str): + # This might not be the right thing in all circumstances + + # Designed to find ansi terminal codes (formatting&color) and santize the text + # '\x1b[0m' => '' # Reset formatting code + # '\x1b[31m' => '' # Red text + ansi_regex = re.compile(r"\x1b\[[0-9;]*m") + ansi_clean = ansi_regex.sub("", text) + # Will turn HTML things back into 'symbols'. E.g. '<' => '<' + decoded = html.unescape(ansi_clean) + return str(decoded.replace("\n", " ")).strip() + + def __post_init__(self): + # Cleaning text + if self.result_text: + self.result_text = self.clean_text(self.result_text) + # Self assertion to double check some mandatory options + # For now this is disabled + + # It's mandatory that the test either partially or fully verifies a requirement + # if self.PartiallyVerifies is None and self.FullyVerifies is None: + # raise ValueError( + # f"TestCase: {self.id} Error. Either 'PartiallyVerifies' or 'FullyVerifies' must be provided." + # ) + # Skipped tests should always have a reason associated with them + # if "skipped" in self.result.keys() and not list(self.result.values())[0]: + # raise ValueError( + # f"TestCase: {self.id} Error. Test was skipped without provided reason, reason is mandatory for skipped tests." + # ) + + def get_test_links(self) -> list[DataForTestLink]: + """Convert TestCaseNeed to list of TestLink objects.""" + + def parse_attributes(self, verify_field: str | None, verify_type: str): + """Process a verification field and yield TestLink objects.""" + if not verify_field: + return + + LOGGER.debug( + f"{verify_type.upper()} VERIFIES: {verify_field}", + type="score_source_code_linker", + ) + + for need in verify_field.split(","): + yield DataForTestLink( + name=self.name, + file=Path(self.file), + line=int(self.line), + need=need.strip(), + verify_type=verify_type, + result=self.result, + result_text=self.result_text, + ) + + return list( + chain( + parse_attributes(self, self.PartiallyVerifies, "partially"), + parse_attributes(self, self.FullyVerifies, "fully"), + ) + ) + + +class DataOfTestCase_JSON_Encoder(json.JSONEncoder): + def default(self, o: object): + if isinstance(o, DataOfTestCase): + return asdict(o) + return super().default(o) + + +def DataOfTestCase_JSON_Decoder(d: dict[str, Any]) -> DataOfTestCase | dict[str, Any]: + if { + "name", + "file", + "line", + "result", + "TestType", + "DerivationTechnique", + "result_text", + "PartiallyVerifies", + "FullyVerifies", + } <= d.keys(): + return DataOfTestCase( + name=d["name"], + file=d["file"], + line=d["line"], + result=d["result"], + TestType=d["TestType"], + DerivationTechnique=d["DerivationTechnique"], + result_text=d["result_text"], + PartiallyVerifies=d["PartiallyVerifies"], + FullyVerifies=d["FullyVerifies"], + ) + # It's something else, pass it on to other decoders + return d + + +def store_test_xml_parsed_json(file: Path, testlist: list[DataForTestLink]): + """ + TestCases that are 'skipped' do not have properties, therefore they will NOT be saved/transformed + to TestLinks. + """ + # After `rm -rf _build` or on clean builds the directory does not exist, so we need to create it + file.parent.mkdir(exist_ok=True) + with open(file, "w") as f: + json.dump( + testlist, + f, + cls=DataForTestLink_JSON_Encoder, + indent=2, + ensure_ascii=False, + ) + + +def load_test_xml_parsed_json(file: Path) -> list[DataForTestLink]: + links: list[DataForTestLink] = json.loads( + file.read_text(encoding="utf-8"), + object_hook=DataForTestLink_JSON_Decoder, + ) + assert isinstance(links, list), ( + "The source xml parser links should be a list of TestLink objects." + ) + assert all(isinstance(link, DataForTestLink) for link in links), ( + "All items in source_xml_parser should be TestLink objects." + ) + return links + + +def store_data_of_test_case_json(file: Path, testneeds: list[DataOfTestCase]): + # After `rm -rf _build` or on clean builds the directory does not exist, so we need to create it + file.parent.mkdir(exist_ok=True) + with open(file, "w") as f: + json.dump( + testneeds, + f, + cls=DataOfTestCase_JSON_Encoder, + indent=2, + ensure_ascii=False, + ) + + +def load_data_of_test_case_json(file: Path) -> list[DataOfTestCase]: + links: list[DataOfTestCase] = json.loads( + file.read_text(encoding="utf-8"), + object_hook=DataOfTestCase_JSON_Decoder, + ) + assert isinstance(links, list), ( + "The test_case_need json should be a list of TestCaseNeed objects." + ) + assert all(isinstance(link, DataOfTestCase) for link in links), ( + "All items in source_xml_parser should be TestCaseNeed objects." + ) + return links diff --git a/src/extensions/score_source_code_linker/tests/scl_golden_file.json b/src/extensions/score_source_code_linker/tests/expected_codelink.json similarity index 97% rename from src/extensions/score_source_code_linker/tests/scl_golden_file.json rename to src/extensions/score_source_code_linker/tests/expected_codelink.json index e5584a12d..457d6a61b 100644 --- a/src/extensions/score_source_code_linker/tests/scl_golden_file.json +++ b/src/extensions/score_source_code_linker/tests/expected_codelink.json @@ -8,7 +8,7 @@ }, { "file": "src/implementation2.py", - "line": 3, + "line": 5, "tag":"#-----req-Id:", "need": "TREQ_ID_1", "full_line": "#-----req-Id: TREQ_ID_1" diff --git a/src/extensions/score_source_code_linker/tests/expected_grouped.json b/src/extensions/score_source_code_linker/tests/expected_grouped.json new file mode 100644 index 000000000..da05343c1 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/expected_grouped.json @@ -0,0 +1,113 @@ +[ + { + "need": "TREQ_ID_1", + "links": { + "CodeLinks": [ + { + "file": "src/implementation1.py", + "line": 3, + "tag":"#-----req-Id:", + "need": "TREQ_ID_1", + "full_line": "#-----req-Id: TREQ_ID_1" + }, + { + "file": "src/implementation2.py", + "line": 5, + "tag":"#-----req-Id:", + "need": "TREQ_ID_1", + "full_line": "#-----req-Id: TREQ_ID_1" + } + + ], + "TestLinks": [ + { + "name": "test_system_startup_time", + "file": "src/tests/testfile_2.py", + "line": 25, + "need": "TREQ_ID_1", + "verify_type": "fully", + "result": "passed", + "result_text": "" + } + ] + } + }, + { + "need": "TREQ_ID_2", + "links": { + "CodeLinks": [ + { + "file": "src/implementation1.py", + "line": 9, + "tag":"#-----req-Id:", + "need": "TREQ_ID_2", + "full_line":"#-----req-Id: TREQ_ID_2" + } + ], + "TestLinks": [ + + { + "name": "test_api_response_format", + "file": "src/testfile_1.py", + "line": 10, + "need": "TREQ_ID_2", + "verify_type": "partially", + "result": "passed", + "result_text": "" + }, + { + "name": "test_error_handling", + "file": "src/testfile_1.py", + "line": 38, + "need": "TREQ_ID_2", + "verify_type": "partially", + "result": "passed", + "result_text": "" + } + + ] + } + }, + { + "need": "TREQ_ID_3", + "links": { + "CodeLinks": [], + "TestLinks": [ + { + "name": "test_api_response_format", + "file": "src/testfile_1.py", + "line": 10, + "need": "TREQ_ID_3", + "verify_type": "partially", + "result": "passed", + "result_text": "" + }, + { + "name": "test_error_handling", + "file": "src/testfile_1.py", + "line": 38, + "need": "TREQ_ID_3", + "verify_type": "partially", + "result": "passed", + "result_text": "" + } + ] + } + }, + { + "need": "TREQ_ID_200", + "links": { + "CodeLinks": [ + { + "file": "src/bad_implementation.py", + "line":2, + "tag":"#-----req-Id:", + "need": "TREQ_ID_200", + "full_line":"#-----req-Id: TREQ_ID_200" + } + + ], + "TestLinks": [] + } + } +] diff --git a/src/extensions/score_source_code_linker/tests/expected_testlink.json b/src/extensions/score_source_code_linker/tests/expected_testlink.json new file mode 100644 index 000000000..9dc32210d --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/expected_testlink.json @@ -0,0 +1,47 @@ +[ + { + "name": "test_api_response_format", + "file": "src/testfile_1.py", + "line": 10, + "need": "TREQ_ID_2", + "verify_type": "partially", + "result": "passed", + "result_text": "" + }, + { + "name": "test_api_response_format", + "file": "src/testfile_1.py", + "line": 10, + "need": "TREQ_ID_3", + "verify_type": "partially", + "result": "passed", + "result_text": "" + }, + { + "name": "test_error_handling", + "file": "src/testfile_1.py", + "line": 38, + "need": "TREQ_ID_2", + "verify_type": "partially", + "result": "passed", + "result_text": "" + }, + { + "name": "test_error_handling", + "file": "src/testfile_1.py", + "line": 38, + "need": "TREQ_ID_3", + "verify_type": "partially", + "result": "passed", + "result_text": "" + }, + { + "name": "test_system_startup_time", + "file": "src/tests/testfile_2.py", + "line": 25, + "need": "TREQ_ID_1", + "verify_type": "fully", + "result": "passed", + "result_text": "" + } +] diff --git a/src/extensions/score_source_code_linker/tests/test_requirement_links.py b/src/extensions/score_source_code_linker/tests/test_codelink.py similarity index 92% rename from src/extensions/score_source_code_linker/tests/test_requirement_links.py rename to src/extensions/score_source_code_linker/tests/test_codelink.py index a6460c1d2..633c01253 100644 --- a/src/extensions/score_source_code_linker/tests/test_requirement_links.py +++ b/src/extensions/score_source_code_linker/tests/test_codelink.py @@ -28,7 +28,6 @@ from src.extensions.score_source_code_linker import ( find_need, get_cache_filename, - get_github_link, group_by_need, ) from src.extensions.score_source_code_linker.needlinks import ( @@ -41,6 +40,7 @@ get_github_repo_info, parse_remote_git_output, ) +from src.helper_lib.additional_functions import get_github_link """ # ────────────────ATTENTION─────────────── @@ -263,7 +263,7 @@ def test_get_cache_filename(): """Test cache filename generation.""" build_dir = Path("/tmp/build") expected = build_dir / "score_source_code_linker_cache.json" - result = get_cache_filename(build_dir) + result = get_cache_filename(build_dir, "score_source_code_linker_cache.json") assert result == expected @@ -333,21 +333,24 @@ def test_group_by_need(sample_needlinks): """Test grouping source code links by need ID.""" result = group_by_need(sample_needlinks) - assert len(result) == 3 - assert len(result["TREQ_ID_1"]) == 2 - assert len(result["TREQ_ID_2"]) == 1 - assert len(result["TREQ_ID_200"]) == 1 - # Check that the grouping is correct - assert result["TREQ_ID_1"][0].file == Path("src/implementation1.py") - assert result["TREQ_ID_1"][1].file == Path("src/implementation2.py") - assert result["TREQ_ID_2"][0].file == Path("src/implementation1.py") - assert result["TREQ_ID_2"][0].line == 9 + assert len(result) == 3 + for found_link in result: + if found_link.need == "TREQ_ID_1": + assert len(found_link.links.CodeLinks) == 2 + assert found_link.links.CodeLinks[0].file == Path("src/implementation1.py") + assert found_link.links.CodeLinks[1].file == Path("src/implementation2.py") + elif found_link.need == "TREQ_ID_2": + assert len(found_link.links.CodeLinks) == 1 + assert found_link.links.CodeLinks[0].file == Path("src/implementation1.py") + assert found_link.links.CodeLinks[0].line == 9 + elif found_link.need == "TREQ_ID_200": + assert len(found_link.links.CodeLinks) == 1 def test_group_by_need_empty_list(): """Test grouping empty list of needlinks.""" - result = group_by_need([]) + result = group_by_need([], []) assert len(result) == 0 @@ -519,17 +522,17 @@ def test_group_by_need_and_find_need_integration(sample_needlinks): ) # Test finding needs for each group - for need_id in grouped: - found_need = find_need(all_needs, need_id, ["PREFIX_"]) - if need_id in ["TREQ_ID_1", "TREQ_ID_2"]: + for found_link in grouped: + found_need = find_need(all_needs, found_link.need, ["PREFIX_"]) + if found_link.need in ["TREQ_ID_1", "TREQ_ID_2"]: assert found_need is not None - assert found_need["id"] == need_id - elif need_id == "TREQ_ID_200": + assert found_need["id"] == found_link.need + elif found_link.need == "TREQ_ID_200": assert found_need is not None assert found_need["id"] == "PREFIX_TREQ_ID_200" -def test_end_to_end_with_real_files(temp_dir, git_repo): +def test_source_linker_end_to_end_with_real_files(temp_dir, git_repo): """Test end-to-end workflow with real files and git repo.""" # Create source files with requirement IDs src_dir = git_repo / "src" @@ -602,8 +605,13 @@ def another_function(): # Test grouping grouped = group_by_need(loaded_links) - assert len(grouped["TREQ_ID_1"]) == 2 - assert len(grouped["TREQ_ID_2"]) == 1 + for found_links in grouped: + if found_links.need == "TREQ_ID_1": + assert len(found_links.links.CodeLinks) == 2 + assert len(found_links.links.TestLinks) == 0 + if found_links.need == "TREQ_ID_2": + assert len(found_links.links.CodeLinks) == 1 + assert len(found_links.links.TestLinks) == 0 # Test GitHub link generation # Have to change directories in order to ensure that we get the right/any .git file diff --git a/src/extensions/score_source_code_linker/tests/test_need_source_links.py b/src/extensions/score_source_code_linker/tests/test_need_source_links.py new file mode 100644 index 000000000..4e1c052cd --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_need_source_links.py @@ -0,0 +1,143 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import pytest + +from src.extensions.score_source_code_linker.need_source_links import ( + NeedSourceLinks, + SourceCodeLinks, + SourceCodeLinks_JSON_Decoder, + SourceCodeLinks_JSON_Encoder, + load_source_code_links_combined_json, + store_source_code_links_combined_json, +) +from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.testlink import DataForTestLink +from src.extensions.score_source_code_linker.tests.test_codelink import ( + NeedLinkTestEncoder, + needlink_test_decoder, +) + + +def SourceCodeLinks_TEST_JSON_Decoder( + d: dict[str, Any], +) -> SourceCodeLinks | dict[str, Any]: + if "need" in d and "links" in d: + links = d["links"] + return SourceCodeLinks( + need=d["need"], + links=NeedSourceLinks( + CodeLinks=[ + needlink_test_decoder(cl) for cl in links.get("CodeLinks", []) + ], + TestLinks=[DataForTestLink(**tl) for tl in links.get("TestLinks", [])], + ), + ) + return d + + +class SourceCodeLinks_TEST_JSON_Encoder(json.JSONEncoder): + def default(self, o: object): + if isinstance(o, SourceCodeLinks): + return { + "need": o.need, + "links": self.default(o.links), + } + if isinstance(o, NeedSourceLinks): + return { + "CodeLinks": [NeedLinkTestEncoder().default(cl) for cl in o.CodeLinks], + "TestLinks": [asdict(tl) for tl in o.TestLinks], + } + if isinstance(o, Path): + return str(o) + return super().default(o) + + +@pytest.fixture +def sample_needlink() -> NeedLink: + return NeedLink( + file=Path("src/example.py"), + line=10, + tag="# req:", + need="REQ_001", + full_line="# req: REQ_001", + ) + + +@pytest.fixture +def sample_testlink() -> DataForTestLink: + return DataForTestLink( + name="test_example", + file=Path("tests/test_example.py"), + need="REQ_001", + line=5, + verify_type="partially", + result="passed", + result_text="", + ) + + +@pytest.fixture +def sample_source_code_links(sample_needlink, sample_testlink) -> SourceCodeLinks: + return SourceCodeLinks( + need="REQ_001", + links=NeedSourceLinks(CodeLinks=[sample_needlink], TestLinks=[sample_testlink]), + ) + + +def test_encoder_outputs_serializable_dict(sample_source_code_links): + encoded = json.dumps(sample_source_code_links, cls=SourceCodeLinks_JSON_Encoder) + assert isinstance(encoded, str) + assert "REQ_001" in encoded + + +def test_decoder_reconstructs_object(sample_source_code_links): + encoded = json.dumps(sample_source_code_links, cls=SourceCodeLinks_JSON_Encoder) + decoded = json.loads(encoded, object_hook=SourceCodeLinks_JSON_Decoder) + assert isinstance(decoded, SourceCodeLinks) + assert decoded.need == "REQ_001" + assert isinstance(decoded.links, NeedSourceLinks) + assert decoded.links.CodeLinks[0].need == "REQ_001" + + +def test_store_and_load_json(tmp_path: Path, sample_source_code_links): + test_file = tmp_path / "combined_links.json" + store_source_code_links_combined_json(test_file, [sample_source_code_links]) + assert test_file.exists() + + loaded = load_source_code_links_combined_json(test_file) + assert isinstance(loaded, list) + assert len(loaded) == 1 + assert isinstance(loaded[0], SourceCodeLinks) + assert loaded[0].need == sample_source_code_links.need + + +def test_load_invalid_json_type(tmp_path: Path): + test_file = tmp_path / "invalid.json" + test_file.write_text('{"not_a_list": true}', encoding="utf-8") + + with pytest.raises(AssertionError, match="should be a list of SourceCodeLinks"): + _ = load_source_code_links_combined_json(test_file) + + +def test_load_invalid_json_items(tmp_path: Path): + test_file = tmp_path / "bad_items.json" + # This is a list but doesn't contain SourceCodeLinks + test_file.write_text('[{"some": "thing"}]', encoding="utf-8") + + with pytest.raises(AssertionError, match="should be SourceCodeLinks objects"): + _ = load_source_code_links_combined_json(test_file) diff --git a/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py new file mode 100644 index 000000000..9f7d2578a --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_source_code_link_integration.py @@ -0,0 +1,553 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +import os +import shutil +import subprocess +from collections import Counter +from collections.abc import Callable +from pathlib import Path +from typing import cast + +import pytest +from pytest import TempPathFactory +from sphinx.testing.util import SphinxTestApp +from sphinx_needs.data import SphinxNeedsData + +from src.extensions.score_source_code_linker.needlinks import NeedLink +from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink, + DataForTestLink_JSON_Decoder, +) +from src.extensions.score_source_code_linker.tests.test_codelink import ( + needlink_test_decoder, +) +from src.extensions.score_source_code_linker.tests.test_need_source_links import ( + SourceCodeLinks_TEST_JSON_Decoder, +) +from src.helper_lib import find_ws_root, get_github_base_url +from src.helper_lib.additional_functions import get_github_link + + +@pytest.fixture() +def sphinx_base_dir(tmp_path_factory: TempPathFactory) -> Path: + repo_path = tmp_path_factory.mktemp("test_git_repo") + return repo_path + + +@pytest.fixture() +def git_repo_setup(sphinx_base_dir) -> Path: + """Creating git repo, to make testing possible""" + + repo_path = sphinx_base_dir + subprocess.run(["git", "init"], cwd=repo_path, check=True) + subprocess.run( + ["git", "config", "user.name", "Test User"], cwd=repo_path, check=True + ) + subprocess.run( + ["git", "config", "user.email", "test@example.com"], cwd=repo_path, check=True + ) + + subprocess.run( + ["git", "remote", "add", "origin", "https://github.com/testorg/testrepo.git"], + cwd=repo_path, + check=True, + ) + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(repo_path) + return repo_path + + +@pytest.fixture() +def create_demo_files(sphinx_base_dir, git_repo_setup): + repo_path = sphinx_base_dir + + # Create some source files with requirement IDs + source_dir = repo_path / "src" + source_dir.mkdir() + + # Create source files that contain requirement references + (source_dir / "implementation1.py").write_text(make_codelink_source_1()) + + (source_dir / "implementation2.py").write_text(make_codelink_source_2()) + (source_dir / "bad_implementation.py").write_text(make_codelink_bad_source()) + # Create a docs directory for Sphinx + docs_dir = repo_path / "docs" + docs_dir.mkdir() + (docs_dir / "index.rst").write_text(basic_needs()) + (docs_dir / "conf.py").write_text(basic_conf()) + + # Create test.xml files + bazel_testdir1 = repo_path / "bazel-testlogs" + bazel_testdir1.mkdir() + bazel_testdir2 = bazel_testdir1 / "src" + bazel_testdir2.mkdir() + + (bazel_testdir2 / "test.xml").write_text(make_test_xml_1()) + testsdir = bazel_testdir2 / "tests" + testsdir.mkdir() + (testsdir / "test.xml").write_text(make_test_xml_2()) + + curr_dir = Path(__file__).absolute().parent + # print("CURR_dir", curr_dir) + shutil.copyfile( + curr_dir / "expected_codelink.json", repo_path / ".expected_codelink.json" + ) + shutil.copyfile( + curr_dir / "expected_testlink.json", repo_path / ".expected_testlink.json" + ) + shutil.copyfile( + curr_dir / "expected_grouped.json", repo_path / ".expected_grouped.json" + ) + + # Add files to git and commit + subprocess.run(["git", "add", "."], cwd=repo_path, check=True) + subprocess.run( + ["git", "commit", "-m", "Initial commit with test files"], + cwd=repo_path, + check=True, + ) + + +def make_codelink_source_1(): + return ( + """ +# This is a test implementation file +#""" + + """ req-Id: TREQ_ID_1 +def some_function(): + pass + +# Some other code here +# More code... +#""" + """ req-Id: TREQ_ID_2 +def another_function(): + pass +""" + ) + + +def make_codelink_source_2(): + return ( + """ +# Another implementation file +# Though we should make sure this +# is at a different line than the other ID +#""" + + """ req-Id: TREQ_ID_1 +class SomeClass: + def method(self): + pass + +""" + ) + + +def make_codelink_bad_source(): + return ( + """ +#""" + + """ req-Id: TREQ_ID_200 +def This_Should_Error(self): + pass + +""" + ) + + +def make_test_xml_1(): + return """ + + + + + + + + + + + + + + + + + + +""" + + +def make_test_xml_2(): + return """ + + + + + + + + + + + This is a message that shouldn't show up + + + +""" + + +def construct_gh_url() -> str: + gh = get_github_base_url() + return f"{gh}/blob/" + + +@pytest.fixture() +def sphinx_app_setup( + sphinx_base_dir, create_demo_files, git_repo_setup +) -> Callable[[], SphinxTestApp]: + def _create_app(): + base_dir = sphinx_base_dir + docs_dir = base_dir / "docs" + + # CRITICAL: Change to a directory that exists and is accessible + # This fixes the "no such file or directory" error in Bazel + original_cwd = None + try: + original_cwd = os.getcwd() + except FileNotFoundError: + # Current working directory doesn't exist, which is the problem + pass + + # Change to the base_dir before creating SphinxTestApp + os.chdir(base_dir) + try: + return SphinxTestApp( + freshenv=True, + srcdir=docs_dir, + confdir=docs_dir, + outdir=sphinx_base_dir / "out", + buildername="html", + warningiserror=True, + ) + finally: + # Try to restore original directory, but don't fail if it doesn't exist + if original_cwd is not None: + try: + os.chdir(original_cwd) + except (FileNotFoundError, OSError): + # Original directory might not exist anymore in Bazel sandbox + pass + + return _create_app + + +def basic_conf(): + return """ +extensions = [ + "sphinx_needs", + "score_source_code_linker", +] +needs_types = [ + dict( + directive="test_req", + title="Testing Requirement", + prefix="TREQ_", + color="#BFD8D2", + style="node", + ), +] +needs_extra_options = ["source_code_link", "testlink"] +needs_extra_links = [{ + "option": "partially_verifies", + "incoming": "paritally_verified_by", + "outgoing": "paritally_verifies", + }, + { + "option": "fully_verifies", + "incoming": "fully_verified_by", + "outgoing": "fully_verifies", + }] + +""" + + +def basic_needs(): + return """ +TESTING SOURCE LINK +=================== + +.. test_req:: TestReq1 + :id: TREQ_ID_1 + :status: valid + +.. test_req:: TestReq2 + :id: TREQ_ID_2 + :status: open + +.. test_req:: TestReq3 + :id: TREQ_ID_3 + :status: open +""" + + +@pytest.fixture() +def example_source_link_text_all_ok(sphinx_base_dir): + repo_path = sphinx_base_dir + return { + "TREQ_ID_1": [ + NeedLink( + file=Path("src/implementation1.py"), + line=3, + tag="#" + " req-Id:", + need="TREQ_ID_1", + full_line="#" + " req-Id: TREQ_ID_1", + ), + NeedLink( + file=Path("src/implementation2.py"), + line=3, + tag="#" + " req-Id:", + need="TREQ_ID_1", + full_line="#" + " req-Id: TREQ_ID_1", + ), + ], + "TREQ_ID_2": [ + NeedLink( + file=Path("src/implementation1.py"), + line=9, + tag="#" + " req-Id:", + need="TREQ_ID_2", + full_line="#" + " req-Id: TREQ_ID_2", + ) + ], + } + + +@pytest.fixture() +def example_test_link_text_all_ok(sphinx_base_dir): + repo_path = sphinx_base_dir + return { + "TREQ_ID_1": [ + DataForTestLink( + name="test_system_startup_time", + file=Path("src/tests/testfile_2.py"), + need="TREQ_ID_1", + line=25, + verify_type="fully", + result="passed", + result_text="", + ), + ], + "TREQ_ID_2": [ + DataForTestLink( + name="test_api_response_format", + file=Path("src/testfile_1.py"), + need="TREQ_ID_2", + line=10, + verify_type="partially", + result="passed", + result_text="", + ), + DataForTestLink( + name="test_error_handling", + file=Path("src/tests/testfile_2.py"), + need="TREQ_ID_2", + line=33, + verify_type="partially", + result="passed", + result_text="", + ), + ], + "TREQ_ID_3": [ + DataForTestLink( + name="test_api_response_format", + file=Path("src/testfile_1.py"), + need="TREQ_ID_3", + line=10, + verify_type="partially", + result="passed", + result_text="", + ), + DataForTestLink( + name="test_error_handling", + file=Path("src/test/testfile_2.py"), + need="TREQ_ID_3", + line=38, + verify_type="partially", + result="passed", + result_text="", + ), + ], + } + + +@pytest.fixture() +def example_source_link_text_non_existent(sphinx_base_dir): + repo_path = sphinx_base_dir + return [ + { + "TREQ_ID_200": [ + NeedLink( + file=Path("src/bad_implementation.py"), + line=2, + tag="#" + " req-Id:", + need="TREQ_ID_200", + full_line="#" + " req-Id: TREQ_ID_200", + ) + ] + } + ] + + +def make_source_link(needlinks): + return ", ".join(f"{get_github_link(n)}<>{n.file}:{n.line}" for n in needlinks) + + +def make_test_link(testlinks): + return ", ".join(f"{get_github_link(n)}<>{n.name}" for n in testlinks) + + +def compare_json_files(file1: Path, expected_file: Path, object_hook): + """Golden File tests with a known good file and the one created""" + with open(file1) as f1: + json1 = json.load(f1, object_hook=object_hook) + with open(expected_file) as f2: + json2 = json.load(f2, object_hook=object_hook) + assert len(json1) == len(json2), ( + f"{file1}'s lenth are not the same as in the golden file lenght. Len of{file1}: {len(json1)}. Len of Golden File: {len(json2)}" + ) + c1 = Counter(n for n in json1) + c2 = Counter(n for n in json2) + assert c1 == c2, ( + f"Testfile does not have same needs as golden file. Testfile: {c1}\nGoldenFile: {c2}" + ) + + +def compare_grouped_json_files(file1: Path, golden_file: Path): + """Golden File tests with a known good file and the one created""" + with open(file1) as f1: + json1 = json.load(f1, object_hook=SourceCodeLinks_TEST_JSON_Decoder) + with open(golden_file) as f2: + json2 = json.load(f2, object_hook=SourceCodeLinks_TEST_JSON_Decoder) + + assert len(json1) == len(json2), ( + f"Input & Expected have different Lenghts. Input: {file1}: {len(json1)}, Expected: {golden_file}: {len(json2)}" + ) + + json1_sorted = sorted(json1, key=lambda x: x.need) + json2_sorted = sorted(json2, key=lambda x: x.need) + + for item1, item2 in zip(json1_sorted, json2_sorted, strict=False): + assert item1.need == item2.need, ( + f"Needs don't match: {item1.need} vs {item2.need}" + ) + + # Need to sort it to make sure we compare content not order + codelinks1_sorted = sorted(item1.links.CodeLinks) + codelinks2_sorted = sorted(item2.links.CodeLinks) + assert codelinks1_sorted == codelinks2_sorted, ( + f"CodeLinks don't match for {item1.need}. " + f"{file1}: {item1.links.CodeLinks}, {golden_file}: {item2.links.CodeLinks}" + ) + + testlinks1_sorted = sorted(item1.links.TestLinks) + testlinks2_sorted = sorted(item2.links.TestLinks) + assert testlinks1_sorted == testlinks2_sorted, ( + f"TestLinks don't match for {item1.need}. " + f"{file1}: {item1.links.TestLinks}, {golden_file}: {item2.links.TestLinks}" + ) + + +def test_source_link_integration_ok( + sphinx_app_setup: Callable[[], SphinxTestApp], + example_source_link_text_all_ok: dict[str, list[str]], + example_test_link_text_all_ok: dict[str, list[str]], + sphinx_base_dir, + git_repo_setup, + create_demo_files, +): + """This is a test description""" + app = sphinx_app_setup() + try: + os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) + app.build() + ws_root = find_ws_root() + if ws_root is None: + # This should never happen + pytest.fail(f"WS_root is none. WS_root: {ws_root}") + Needs_Data = SphinxNeedsData(app.env) + needs_data = {x["id"]: x for x in Needs_Data.get_needs_view().values()} + compare_json_files( + app.outdir / "score_source_code_linker_cache.json", + sphinx_base_dir / ".expected_codelink.json", + needlink_test_decoder, + ) + compare_json_files( + app.outdir / "score_xml_parser_cache.json", + sphinx_base_dir / ".expected_testlink.json", + DataForTestLink_JSON_Decoder, + ) + compare_grouped_json_files( + app.outdir / "score_scl_grouped_cache.json", + sphinx_base_dir / ".expected_grouped.json", + ) + # Testing TREQ_ID_1, TREQ_ID_2, TREQ_ID_3 + + # TODO: Is this actually a good test, or just a weird mock? + for i in range(1, 4): + # extra_options are only available at runtime + assert f"TREQ_ID_{i}" in needs_data + need_as_dict = cast(dict[str, object], needs_data[f"TREQ_ID_{i}"]) + # TODO: This probably isn't great. Should make this better. + if i != 3: + # Excluding 3 as this is a keyerror here + expected_code_link = make_source_link( + example_source_link_text_all_ok[f"TREQ_ID_{i}"] + ) + print(f"EXPECTED LINK CODE: {expected_code_link}") + actual_source_code_link = cast( + list[str], need_as_dict["source_code_link"] + ) + print(f"ACTUALL CODE LINK: {actual_source_code_link}") + assert set(expected_code_link) == set(actual_source_code_link) + expected_test_link = make_test_link( + example_test_link_text_all_ok[f"TREQ_ID_{i}"] + ) + # Compare contents, regardless of order. + print(f"NEED AS DICT: {need_as_dict}") + print(f"EXPECTED LINK TEST: {expected_test_link}") + actual_test_code_link = cast(list[str], need_as_dict["testlink"]) + print(f"ACTUALL TEST LINK: {actual_test_code_link}") + assert set(expected_test_link) == set(actual_test_code_link) + finally: + app.cleanup() + + +def test_source_link_integration_non_existent_id( + sphinx_app_setup: Callable[[], SphinxTestApp], + example_source_link_text_non_existent: dict[str, list[str]], + sphinx_base_dir, + git_repo_setup, + create_demo_files, +): + """Asserting warning if need not found""" + app = sphinx_app_setup() + try: + app.build() + warnings = app.warning.getvalue() + assert ( + "src/bad_implementation.py:2: Could not find TREQ_ID_200 in documentation" + in warnings + ) + finally: + app.cleanup() diff --git a/src/extensions/score_source_code_linker/tests/test_source_link.py b/src/extensions/score_source_code_linker/tests/test_source_link.py deleted file mode 100644 index 32c022f0d..000000000 --- a/src/extensions/score_source_code_linker/tests/test_source_link.py +++ /dev/null @@ -1,336 +0,0 @@ -# ******************************************************************************* -# Copyright (c) 2025 Contributors to the Eclipse Foundation -# -# See the NOTICE file(s) distributed with this work for additional -# information regarding copyright ownership. -# -# This program and the accompanying materials are made available under the -# terms of the Apache License Version 2.0 which is available at -# https://www.apache.org/licenses/LICENSE-2.0 -# -# SPDX-License-Identifier: Apache-2.0 -# ******************************************************************************* -import contextlib -import json -import os -import shutil -import subprocess -from collections import Counter -from collections.abc import Callable -from pathlib import Path -from typing import cast - -import pytest -from pytest import TempPathFactory -from sphinx.testing.util import SphinxTestApp -from sphinx_needs.data import SphinxNeedsData -from test_requirement_links import needlink_test_decoder - -from src.extensions.score_source_code_linker import get_github_base_url, get_github_link -from src.extensions.score_source_code_linker.needlinks import NeedLink -from src.helper_lib import find_ws_root - - -@pytest.fixture() -def sphinx_base_dir(tmp_path_factory: TempPathFactory) -> Path: - return tmp_path_factory.mktemp("test_git_repo") - - -@pytest.fixture() -def git_repo_setup(sphinx_base_dir) -> Path: - """Creating git repo, to make testing possible""" - - repo_path = sphinx_base_dir - subprocess.run(["git", "init"], cwd=repo_path, check=True) - subprocess.run( - ["git", "config", "user.name", "Test User"], cwd=repo_path, check=True - ) - subprocess.run( - ["git", "config", "user.email", "test@example.com"], cwd=repo_path, check=True - ) - - subprocess.run( - ["git", "remote", "add", "origin", "https://github.com/testorg/testrepo.git"], - cwd=repo_path, - check=True, - ) - os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(repo_path) - return repo_path - - -@pytest.fixture() -def create_demo_files(sphinx_base_dir, git_repo_setup): - repo_path = sphinx_base_dir - - # Create some source files with requirement IDs - source_dir = repo_path / "src" - source_dir.mkdir() - - # Create source files that contain requirement references - (source_dir / "implementation1.py").write_text(make_source_1()) - - (source_dir / "implementation2.py").write_text(make_source_2()) - (source_dir / "bad_implementation.py").write_text(make_bad_source()) - # Create a docs directory for Sphinx - docs_dir = repo_path / "docs" - docs_dir.mkdir() - (docs_dir / "index.rst").write_text(basic_needs()) - (docs_dir / "conf.py").write_text(basic_conf()) - curr_dir = Path(__file__).absolute().parent - # print("CURR_dir", curr_dir) - shutil.copyfile(curr_dir / "scl_golden_file.json", repo_path / ".golden_file.json") - - # Add files to git and commit - subprocess.run(["git", "add", "."], cwd=repo_path, check=True) - subprocess.run( - ["git", "commit", "-m", "Initial commit with test files"], - cwd=repo_path, - check=True, - ) - - # Cleanup - # Don't know if we need this? - # os.environ.pop("BUILD_WORKSPACE_DIRECTORY", None) - - -def make_source_1(): - return ( - """ -# This is a test implementation file -#""" - + """ req-Id: TREQ_ID_1 -def some_function(): - pass - -# Some other code here -# More code... -#""" - """ req-Id: TREQ_ID_2 -def another_function(): - pass -""" - ) - - -def make_source_2(): - return ( - """ -# Another implementation file -#""" - + """ req-Id: TREQ_ID_1 -class SomeClass: - def method(self): - pass - -""" - ) - - -def make_bad_source(): - return ( - """ -#""" - + """ req-Id: TREQ_ID_200 -def This_Should_Error(self): - pass - -""" - ) - - -def construct_gh_url() -> str: - gh = get_github_base_url() - return f"{gh}/blob/" - - -@pytest.fixture() -def sphinx_app_setup( - sphinx_base_dir, create_demo_files, git_repo_setup -) -> Callable[[], SphinxTestApp]: - def _create_app(): - base_dir = sphinx_base_dir - docs_dir = base_dir / "docs" - - original_cwd = None - # CRITICAL: Change to a directory that exists and is accessible - # This fixes the "no such file or directory" error in Bazel - with contextlib.suppress(FileNotFoundError): - original_cwd = os.getcwd() - - # Change to the base_dir before creating SphinxTestApp - os.chdir(base_dir) - try: - return SphinxTestApp( - freshenv=True, - srcdir=docs_dir, - confdir=docs_dir, - outdir=sphinx_base_dir / "out", - buildername="html", - warningiserror=True, - ) - finally: - # Try to restore original directory, but don't fail if it doesn't exist - if original_cwd is not None: - # Original directory might not exist anymore in Bazel sandbox - with contextlib.suppress(FileNotFoundError, OSError): - os.chdir(original_cwd) - - return _create_app - - -def basic_conf(): - return """ -extensions = [ - "sphinx_needs", - "score_source_code_linker", -] -needs_types = [ - dict( - directive="test_req", - title="Testing Requirement", - prefix="TREQ_", - color="#BFD8D2", - style="node", - ), -] -needs_extra_options = ["source_code_link"] -""" - - -def basic_needs(): - return """ -TESTING SOURCE LINK -=================== - -.. test_req:: TestReq1 - :id: TREQ_ID_1 - :status: valid - -.. test_req:: TestReq2 - :id: TREQ_ID_2 - :status: open -""" - - -@pytest.fixture() -def example_source_link_text_all_ok(sphinx_base_dir): - return { - "TREQ_ID_1": [ - NeedLink( - file=Path("src/implementation1.py"), - line=3, - tag="#" + " req-Id:", - need="TREQ_ID_1", - full_line="#" + " req-Id: TREQ_ID_1", - ), - NeedLink( - file=Path("src/implementation2.py"), - line=3, - tag="#" + " req-Id:", - need="TREQ_ID_1", - full_line="#" + " req-Id: TREQ_ID_1", - ), - ], - "TREQ_ID_2": [ - NeedLink( - file=Path("src/implementation1.py"), - line=9, - tag="#" + " req-Id:", - need="TREQ_ID_2", - full_line="#" + " req-Id: TREQ_ID_2", - ) - ], - } - - -@pytest.fixture() -def example_source_link_text_non_existent(sphinx_base_dir): - return [ - { - "TREQ_ID_200": [ - NeedLink( - file=Path("src/bad_implementation.py"), - line=2, - tag="#" + " req-Id:", - need="TREQ_ID_200", - full_line="#" + " req-Id: TREQ_ID_200", - ) - ] - } - ] - - -def make_source_link(needlinks): - return ", ".join(f"{get_github_link(n)}<>{n.file}:{n.line}" for n in needlinks) - - -def compare_json_files(file1: Path, golden_file: Path): - with open(file1) as f1: - json1 = json.load(f1, object_hook=needlink_test_decoder) - with open(golden_file) as f2: - json2 = json.load(f2, object_hook=needlink_test_decoder) - assert len(json1) == len(json2), ( - f"{file1}'s lenth are not the same as in the golden file lenght. " - f"Len of{file1}: {len(json1)}. Len of Golden File: {len(json2)}" - ) - c1 = Counter(n for n in json1) - c2 = Counter(n for n in json2) - assert c1 == c2, ( - "Testfile does not have same needs as golden file. " - f"Testfile: {c1}\nGoldenFile: {c2}" - ) - - -def test_source_link_integration_ok( - sphinx_app_setup: Callable[[], SphinxTestApp], - example_source_link_text_all_ok: dict[str, list[str]], - sphinx_base_dir, - git_repo_setup, - create_demo_files, -): - app = sphinx_app_setup() - try: - os.environ["BUILD_WORKSPACE_DIRECTORY"] = str(sphinx_base_dir) - app.build() - ws_root = find_ws_root() - if ws_root is None: - # This should never happen - pytest.fail(f"WS_root is none. WS_root: {ws_root}") - Needs_Data = SphinxNeedsData(app.env) - needs_data = {x["id"]: x for x in Needs_Data.get_needs_view().values()} - compare_json_files( - app.outdir / "score_source_code_linker_cache.json", - sphinx_base_dir / ".golden_file.json", - ) - # Testing TREQ_ID_1 & TREQ_ID_2 - for i in range(1, 3): - assert f"TREQ_ID_{i}" in needs_data - need_as_dict = cast(dict[str, object], needs_data[f"TREQ_ID_{i}"]) - expected_link = make_source_link( - example_source_link_text_all_ok[f"TREQ_ID_{i}"] - ) - # extra_options are only available at runtime - # Compare contents, regardless of order. - actual_source_code_link = cast(list[str], need_as_dict["source_code_link"]) - assert set(expected_link) == set(actual_source_code_link) - finally: - app.cleanup() - - -def test_source_link_integration_non_existent_id( - sphinx_app_setup: Callable[[], SphinxTestApp], - example_source_link_text_non_existent: dict[str, list[str]], - sphinx_base_dir, - git_repo_setup, - create_demo_files, -): - app = sphinx_app_setup() - try: - app.build() - warnings = app.warning.getvalue() - assert ( - "src/bad_implementation.py:2: Could not find TREQ_ID_200 in documentation" - in warnings - ) - finally: - app.cleanup() diff --git a/src/extensions/score_source_code_linker/tests/test_testlink.py b/src/extensions/score_source_code_linker/tests/test_testlink.py new file mode 100644 index 000000000..09e08d254 --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_testlink.py @@ -0,0 +1,118 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +import json +from pathlib import Path + +from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink, + DataForTestLink_JSON_Decoder, + DataForTestLink_JSON_Encoder, + DataOfTestCase, + load_test_xml_parsed_json, + store_test_xml_parsed_json, +) + + +def test_testlink_serialization_roundtrip(): + link = DataForTestLink( + name="my_test", + file=Path("some/file.py"), + line=123, + need="REQ_001", + verify_type="fully", + result="passed", + result_text="All good", + ) + dumped = json.dumps(link, cls=DataForTestLink_JSON_Encoder) + loaded = json.loads(dumped, object_hook=DataForTestLink_JSON_Decoder) + + assert isinstance(loaded, DataForTestLink) + assert loaded == link + + +def test_testlink_encoder_handles_path(): + data = {"file": Path("some/thing.py")} + encoded = json.dumps(data, cls=DataForTestLink_JSON_Encoder) + assert '"file": "some/thing.py"' in encoded + + +def test_decoder_ignores_irrelevant_dicts(): + input_data = {"foo": "bar"} + result = DataForTestLink_JSON_Decoder(input_data) + assert result == input_data + + +def test_clean_text_removes_ansi_and_html_unescapes(): + raw = "\x1b[31m<b>Warning</b>\x1b[0m\nExtra line" + cleaned = DataOfTestCase.clean_text(raw) + assert cleaned == "Warning Extra line" + + +def test_testcaseneed_to_dict_multiple_links(): + case = DataOfTestCase( + name="TC_01", + file="src/test.py", + line="10", + result="failed", + TestType="unit", + DerivationTechnique="manual", + result_text="Something went wrong", + PartiallyVerifies="REQ-1, REQ-2", + FullyVerifies="REQ-3", + ) + + links = case.get_test_links() + + assert len(links) == 3 + need_ids = [link.need for link in links] + assert set(need_ids) == {"REQ-1", "REQ-2", "REQ-3"} + + for link in links: + assert link.file == Path("src/test.py") + assert link.line == 10 + assert link.name == "TC_01" + assert link.result == "failed" + + +def test_store_and_load_testlinks_roundtrip(tmp_path): + file = tmp_path / "testlinks.json" + + links = [ + DataForTestLink( + name="L1", + file=Path("abc.py"), + line=1, + need="REQ_A", + verify_type="partially", + result="passed", + result_text="Looks good", + ), + DataForTestLink( + name="L2", + file=Path("def.py"), + line=2, + need="REQ_B", + verify_type="fully", + result="failed", + result_text="Needs work", + ), + ] + + store_test_xml_parsed_json(file, links) + assert file.exists() + + reloaded = load_test_xml_parsed_json(file) + + assert reloaded == links + for link in reloaded: + assert isinstance(link, DataForTestLink) diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py new file mode 100644 index 000000000..c87e7947d --- /dev/null +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -0,0 +1,146 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +Tests for the xml_parser.py file. +Keep in mind that this is with the 'assertions' inside xml_parser disabled so far. +Once we enable those we will need to change the tests +""" + +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any + +import pytest + +import src.extensions.score_source_code_linker.xml_parser as xml_parser +from src.extensions.score_source_code_linker.testlink import DataOfTestCase + + +# Unsure if I should make these last a session or not +@pytest.fixture +def tmp_xml_dirs(tmp_path): + root = tmp_path / "bazel-testlogs" + dir1 = root / "with_props" + dir2 = root / "no_props" + dir1.mkdir(parents=True) + dir2.mkdir(parents=True) + + def write(file_path: Path, testcases: list[ET.Element]): + ts = ET.Element("testsuites") + suite = ET.SubElement(ts, "testsuite") + for tc in testcases: + suite.append(tc) + tree = ET.ElementTree(ts) + tree.write(file_path, encoding="utf-8", xml_declaration=True) + + def make_tc( + name: str, + result: str = "", + props: dict[str, str] = dict(), + file: str = "", + line: int = 0, + ): + tc = ET.Element("testcase", {"name": name}) + if file: + tc.set("file", file) + if line: + tc.set("line", str(line)) + if result == "failed": + ET.SubElement(tc, "failure", {"message": "failmsg"}) + elif result == "skipped": + ET.SubElement(tc, "skipped", {"message": "skipmsg"}) + if props: + props_el = ET.SubElement(tc, "properties") + for k, v in props.items(): + ET.SubElement(props_el, "property", {"name": k, "value": v}) + return tc + + # File with properties + tc1 = make_tc( + "tc_with_props", + result="failed", + props={ + "PartiallyVerifies": "REQ1", + "FullyVerifies": "", + "TestType": "type", + "DerivationTechnique": "tech", + "Description": "desc", + }, + file="path1", + line=10, + ) + write(dir1 / "test.xml", [tc1]) + + # File without properties + # HINT: Once the assertions in xml_parser are back and active, this should allow us to catch that the tests + # Need to be changed too. + tc2 = make_tc("tc_no_props", file="path2", line=20) + write(dir2 / "test.xml", [tc2]) + + return root, dir1, dir2 + + +def test_find_xml_files(tmp_xml_dirs): + root, dir1, dir2 = tmp_xml_dirs + found = xml_parser.find_xml_files(root) + expected = {dir1 / "test.xml", dir2 / "test.xml"} + assert set(found) == expected + + +def test_parse_testcase_result(): + tc = ET.Element("testcase", {"name": "a"}) + assert xml_parser.parse_testcase_result(tc) == ("passed", "") + + tc2 = ET.Element("testcase", {"name": "b", "status": "notrun"}) + assert xml_parser.parse_testcase_result(tc2) == ("disabled", "") + + tc3 = ET.Element("testcase", {"name": "c"}) + ET.SubElement(tc3, "failure", {"message": "err"}) + assert xml_parser.parse_testcase_result(tc3) == ("failed", "err") + + tc4 = ET.Element("testcase", {"name": "d"}) + ET.SubElement(tc4, "skipped", {"message": "skp"}) + assert xml_parser.parse_testcase_result(tc4) == ("skipped", "skp") + + +def test_parse_properties(): + cp: dict[str, Any] = {} + props_el = ET.Element("properties") + ET.SubElement(props_el, "property", {"name": "A", "value": "1"}) + ET.SubElement(props_el, "property", {"name": "Description", "value": "ignored"}) + res = xml_parser.parse_properties(cp, props_el) + assert res["A"] == "1" + assert "Description" not in res + + +def test_read_test_xml_file(tmp_xml_dirs): + root, dir1, dir2 = tmp_xml_dirs + + needs1, no_props1 = xml_parser.read_test_xml_file(dir1 / "test.xml") + assert isinstance(needs1, list) and len(needs1) == 1 + tcneed = needs1[0] + assert isinstance(tcneed, DataOfTestCase) + assert tcneed.result == "failed" + assert no_props1 == [] + + needs2, no_props2 = xml_parser.read_test_xml_file(dir2 / "test.xml") + assert needs2 == [] + assert no_props2 == ["tc_no_props"] + + +def test_short_hash_consistency_and_format(): + h1 = xml_parser.short_hash("foo") + h2 = xml_parser.short_hash("foo") + assert h1 == h2 + assert h1.isalpha() + assert len(h1) == 5 diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py new file mode 100644 index 000000000..9ef65bc45 --- /dev/null +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -0,0 +1,241 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +""" +This file deals with finding and parsing of test.xml files that get created during `bazel test`. +It also generates external needs out of the parsed testcases to enable linking to requirements &gathering statistics +""" + +import contextlib +import base64 +import hashlib +import itertools +import os +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any +from xml.etree.ElementTree import Element + +from sphinx.application import Sphinx +from sphinx.environment import BuildEnvironment +from sphinx_needs import logging +from sphinx_needs.api import add_external_need + +from src.extensions.score_source_code_linker.testlink import ( + DataOfTestCase, + store_data_of_test_case_json, + store_test_xml_parsed_json, +) +from src.helper_lib import find_ws_root +from src.helper_lib.additional_functions import get_github_link + +logger = logging.get_logger(__name__) +logger.setLevel("DEBUG") + + +def parse_testcase_result(testcase: ET.Element) -> tuple[str, str]: + """ + Returns 'result' and 'result_text' found in the 'message' + attribute of the result. + Example: + => + + Returns: + ("skipped", "Test skip message") + """ + skipped = testcase.find("skipped") + failed = testcase.find("failure") + status = testcase.get("status") + # NOTE: Special CPP case of 'disabled' + if status is not None and status == "notrun": + return "disabled", "" + if skipped is None and failed is None: + return "passed", "" + if failed is not None: + return "failed", failed.get("message", "") + if skipped is not None: + return "skipped", skipped.get("message", "") + # TODO: Test all possible permuations of this to find if this is unreachable + raise ValueError( + f"Testcase: {testcase.get('name')}. Did not find 'failed', 'skipped' or 'passed' in test" + ) + + +def parse_properties(case_properties: dict[str, Any], properties: Element): + for prop in properties: + prop_name = prop.get("name", "") + prop_value = prop.get("value", "") + # We ignore the Description of the test as a 'property'. + # Every language just needs to ensure each test does have a description. No matter where this resides. + if prop_name == "Description": + continue + case_properties[prop_name] = prop_value + return case_properties + + +def read_test_xml_file(file: Path) -> tuple[list[DataOfTestCase], list[str]]: + """ + Reading & parsing the test.xml files into TestCaseNeeds + + Returns: + tuple consisting of: + - list[TestCaseNeed] + - list[str] => Testcase Names that did not have the required properties. + """ + test_case_needs: list[DataOfTestCase] = [] + non_prop_tests: list[str] = [] + tree = ET.parse(file) + root = tree.getroot() + + for testsuite in root.findall("testsuite"): + for testcase in testsuite.findall("testcase"): + case_properties = {} + testname = testcase.get("name") + assert testname is not None, ( + f"Testcase: {testcase} does not have a 'name' attribute. This is mandatory. This should not happen, something is wrong." + ) + test_file = testcase.get("file") + line = testcase.get("line") + + # ╭──────────────────────────────────────╮ + # │ Assert worldview that mandatory │ + # │ things are actually there │ + # │ Disabled temporarily │ + # ╰──────────────────────────────────────╯ + + # assert test_file is not None, ( + # f"Testcase: {testname} does not have a 'file' attribute. This is mandatory" + # ) + # assert lineNr is not None, ( + # f"Testcase: {testname} located in {test_file} does not have a 'lineNr' attribute. This is mandator" + # ) + case_properties["name"] = testname + case_properties["file"] = test_file + case_properties["line"] = line + case_properties["result"], case_properties["result_text"] = ( + parse_testcase_result(testcase) + ) + + properties_element = testcase.find("properties") + # HINT: This list is hard coded here, might not be ideal to have that in the long run. + if properties_element is None: + non_prop_tests.append(testname) + continue + + # ╓ ╖ + # ║ Disabled Temporarily ║ + # ╙ ╜ + # assert properties_element is not None, ( + # f"Testcase: {testname} located in {test_file}:{lineNr}, does not have any properties. Properties 'TestType', 'DerivationTechnique' and either 'PartiallyVerifies' or 'FullyVerifies' are mandatory." + # ) + + case_properties = parse_properties(case_properties, properties_element) + test_case_needs.append(DataOfTestCase.from_dict(case_properties)) + return test_case_needs, non_prop_tests + + +def find_xml_files(dir: Path) -> list[Path]: + """ + Recursively search all test.xml files inside 'bazel-testlogs' + + Returns: + - list[Path] => Paths to all found 'test.xml' files. + """ + + test_file_name = "test.xml" + + xml_paths: list[Path] = [] + for root, _, files in os.walk(dir): + if test_file_name in files: + xml_paths.append(Path(os.path.join(root, test_file_name))) + return xml_paths + + +def run_xml_parser(app: Sphinx, env: BuildEnvironment): + """ + This is the 'main' function for parsing test.xml's and + building testcase needs. + It gets called from the source_code_linker __init__ + """ + ws_root = find_ws_root() + assert ws_root is not None + bazel_testlogs = ws_root / "bazel-testlogs" + xml_file_paths = find_xml_files(bazel_testlogs) + test_case_needs = build_test_needs_from_files(app, env, xml_file_paths) + # Saving the test case needs for cache + store_data_of_test_case_json( + app.outdir / "score_testcaseneeds_cache.json", test_case_needs + ) + output = list( + itertools.chain.from_iterable(tcn.get_test_links() for tcn in test_case_needs) + ) + # This is not ideal, due to duplication, but I can't think of a better solution right now + store_test_xml_parsed_json(app.outdir / "score_xml_parser_cache.json", output) + + +def build_test_needs_from_files( + app: Sphinx, env: BuildEnvironment, xml_paths: list[Path] +) -> list[DataOfTestCase]: + """ + Reading in all test.xml files, and building 'testcase' external need objects out of them. + + Returns: + - list[TestCaseNeed] + """ + tcns: list[DataOfTestCase] = [] + for f in xml_paths: + b, z = read_test_xml_file(f) + for non_prop_test in z: + # We probably do not want to do this as a warning yet + logger.info( + f"Test: {non_prop_test} has no properties. Could not create need" + ) + # Now we build the needs from it + tcns.extend(b) + for c in b: + construct_and_add_need(app, c) + return tcns + + +def short_hash(input_str: str, length: int = 5) -> str: + # Get a stable hash + sha256 = hashlib.sha256(input_str.encode()).digest() + # Encode to base32 (A-Z + 2-7), decode to str, remove padding + b32 = base64.b32encode(sha256).decode("utf-8").rstrip("=") + # Keep only alphabetic characters + letters_only = "".join(filter(str.isalpha, b32)) + # Return the first `length` letters + return letters_only[:length].lower() + + +def construct_and_add_need(app: Sphinx, tn: DataOfTestCase): + # IDK if this is ideal or not + with contextlib.suppress(BaseException): + _ = add_external_need( + app=app, + need_type="testcase", + title=tn.name, + tags="TEST", + id=f"testcase__{tn.name}_{short_hash(tn.file + tn.name).upper()}", + name=tn.name, + external_url=get_github_link(tn), + fully_verifies=tn.FullyVerifies if tn.FullyVerifies is not None else "", + partially_verifies=tn.PartiallyVerifies + if tn.PartiallyVerifies is not None + else "", + test_type=tn.TestType, + derivation_technique=tn.DerivationTechnique, + file=tn.file, + line=tn.line, + result=tn.result, # We just want the 'failed' or whatever + result_text=tn.result_text if tn.result_text else "", + ) diff --git a/src/helper_lib/BUILD b/src/helper_lib/BUILD index 8e0e16b26..ac51fc2f3 100644 --- a/src/helper_lib/BUILD +++ b/src/helper_lib/BUILD @@ -16,9 +16,13 @@ load("@score_python_basics//:defs.bzl", "score_py_pytest") py_library( name = "helper_lib", - srcs = ["__init__.py"], + srcs = [ + "__init__.py", + "additional_functions.py", + ], imports = ["."], visibility = ["//visibility:public"], + deps = ["@score_docs_as_code//src/extensions/score_source_code_linker:source_code_linker_helpers"], ) score_py_pytest( diff --git a/src/helper_lib/additional_functions.py b/src/helper_lib/additional_functions.py new file mode 100644 index 000000000..5b1ce6d98 --- /dev/null +++ b/src/helper_lib/additional_functions.py @@ -0,0 +1,38 @@ +# ******************************************************************************* +# Copyright (c) 2025 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from pathlib import Path + +# Import types that depend on score_source_code_linker +from src.extensions.score_source_code_linker.needlinks import DefaultNeedLink, NeedLink +from src.extensions.score_source_code_linker.testlink import ( + DataForTestLink, + DataOfTestCase, +) +from src.helper_lib import ( + find_git_root, + get_current_git_hash, + get_github_base_url, +) + + +def get_github_link( + link: NeedLink | DataForTestLink | DataOfTestCase | None = None, +) -> str: + if link is None: + link = DefaultNeedLink() + passed_git_root = find_git_root() + if passed_git_root is None: + passed_git_root = Path() + base_url = get_github_base_url() + current_hash = get_current_git_hash(passed_git_root) + return f"{base_url}/blob/{current_hash}/{link.file}#L{link.line}" diff --git a/src/tests/test_consumer.py b/src/tests/test_consumer.py index 36ce33656..ef1dbda88 100644 --- a/src/tests/test_consumer.py +++ b/src/tests/test_consumer.py @@ -23,8 +23,7 @@ from rich.console import Console from rich.table import Table -from src.extensions.score_source_code_linker import get_github_base_url -from src.helper_lib import find_git_root +from src.helper_lib import find_git_root, get_github_base_url """ This script's main usecase is to test consumers of Docs-As-Code with