Fixing copilot & other findings

Author: MaximilianSoerenPollak

docs.bzl

@@ -183,88 +183,79 @@ def docs(source_dir = "docs", data = [], deps = [], scan_code = [], known_good =
     data_with_docs_sources = _rewrite_needs_json_to_docs_sources(data)
     additional_combo_sourcelinks = _rewrite_needs_json_to_sourcelinks(data)
     _merge_sourcelinks(name = "merged_sourcelinks", sourcelinks = [":sourcelinks_json"] + additional_combo_sourcelinks, known_good = known_good)
+    docs_data = data + [":sourcelinks_json"]
+    combo_data = data_with_docs_sources + [":merged_sourcelinks"]
+
+    docs_env = {
+        "SOURCE_DIRECTORY": source_dir,
+        "DATA": str(data),
+        "ACTION": "incremental",
+        "SCORE_SOURCELINKS": "$(location :sourcelinks_json)",
+    }
+    docs_sources_env = {
+        "SOURCE_DIRECTORY": source_dir,
+        "DATA": str(data_with_docs_sources),
+        "ACTION": "incremental",
+        "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)",
+    }
+    if known_good:
+        docs_env["KNOWN_GOOD_JSON"] = "$(location "+ known_good + ")"
+        docs_sources_env["KNOWN_GOOD_JSON"] = "$(location "+ known_good + ")"
+        docs_data.append(known_good)
+        combo_data.append(known_good)
 
     py_binary(
         name = "docs",
         tags = ["cli_help=Build documentation:\nbazel run //:docs"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data + [":sourcelinks_json"],
+        data = docs_data,
         deps = deps,
-        env = {
-            "SOURCE_DIRECTORY": source_dir,
-            "DATA": str(data),
-            "ACTION": "incremental",
-            "SCORE_SOURCELINKS": "$(location :sourcelinks_json)",
-        },
+        env = docs_env,
     )
 
     py_binary(
         name = "docs_combo_experimental",
         tags = ["cli_help=Build full documentation with all dependencies:\nbazel run //:docs_combo_experimental"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data_with_docs_sources + [":merged_sourcelinks"],
+        data = combo_data,
         deps = deps,
-        env = {
-            "SOURCE_DIRECTORY": source_dir,
-            "DATA": str(data_with_docs_sources),
-            "ACTION": "incremental",
-            "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)",
-        },
+        env = docs_sources_env
     )
 
     py_binary(
         name = "docs_link_check",
         tags = ["cli_help=Verify Links inside Documentation:\nbazel run //:link_check\n (Note: this could take a long time)"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data,
+        data = docs_data,
         deps = deps,
-        env = {
-            "SOURCE_DIRECTORY": source_dir,
-            "DATA": str(data),
-            "ACTION": "linkcheck",
-        },
+        env = docs_env
     )
 
     py_binary(
         name = "docs_check",
         tags = ["cli_help=Verify documentation:\nbazel run //:docs_check"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data + [":sourcelinks_json"],
+        data = docs_data,
         deps = deps,
-        env = {
-            "SOURCE_DIRECTORY": source_dir,
-            "DATA": str(data),
-            "ACTION": "check",
-            "SCORE_SOURCELINKS": "$(location :sourcelinks_json)",
-        },
+        env = docs_env,
     )
 
     py_binary(
         name = "live_preview",
         tags = ["cli_help=Live preview documentation in the browser:\nbazel run //:live_preview"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data + [":sourcelinks_json"],
+        data = docs_data,
         deps = deps,
-        env = {
-            "SOURCE_DIRECTORY": source_dir,
-            "DATA": str(data),
-            "ACTION": "live_preview",
-            "SCORE_SOURCELINKS": "$(location :sourcelinks_json)",
-        },
+        env = docs_env,
     )
 
     py_binary(
         name = "live_preview_combo_experimental",
         tags = ["cli_help=Live preview full documentation with all dependencies in the browser:\nbazel run //:live_preview_combo_experimental"],
         srcs = ["@score_docs_as_code//src:incremental.py"],
-        data = data_with_docs_sources + [":merged_sourcelinks"],
+        data = combo_data,
         deps = deps,
-        env = {
-            "SOURCE_DIRECTORY": source_dir,
-            "DATA": str(data_with_docs_sources),
-            "ACTION": "live_preview",
-            "SCORE_SOURCELINKS": "$(location :merged_sourcelinks)",
-        },
+        env = docs_sources_env
     )
 
     score_virtualenv(

scripts_bazel/merge_sourcelinks.py

@@ -34,6 +34,7 @@
 if external => parse thing for module_name => look up known_good json for hash & url
 """
 
+
 def main():
     parser = argparse.ArgumentParser(
         description="Merge multiple sourcelinks JSON files into one"
@@ -63,7 +64,18 @@ def main():
     for json_file in all_files:
         with open(json_file) as f:
             data = json.load(f)
+            # If the file is empty e.g. '[]' there is nothing to parse, we continue
+            if not data:
+                continue
             metadata = data[0]
+            if not isinstance(metadata, dict) or "module_name" not in metadata:
+                logger.warning(
+                    f"Unexpected schema in sourcelinks file '{json_file}': "
+                    "expected first element to be a metadata dict "
+                    "with a 'module_name' key. "
+                )
+                # As we can't deal with bad JSON structure we just skip it
+                continue
             if metadata["module_name"] and metadata["module_name"] != "local_module":
                 hash, repo = parse_info_from_known_good(
                     known_good_json=args.known_good, module_name=metadata["module_name"]

src/extensions/score_source_code_linker/BUILD

@@ -55,6 +55,7 @@ py_library(
         "testlink.py",
         "xml_parser.py",
         "helpers.py",
+        "module_source_links.py",
     ],
     imports = ["."],
     visibility = ["//visibility:public"],

src/extensions/score_source_code_linker/helpers.py

@@ -57,7 +57,7 @@ def get_github_link_from_git(
 def get_github_link_from_json(
     metadata: ModuleInfo,
     link: NeedLink | DataForTestLink | DataOfTestCase | None = None,
-):
+) -> str:
     if link is None:
         link = DefaultNeedLink()
     base_url = metadata.url
@@ -67,24 +67,24 @@ def get_github_link_from_json(
 
 def parse_module_name_from_path(path: Path) -> str:
     """
-    Parse out the Module-Name from the filename gotten
-    /home/user/.cache/bazel/aksj37981712/external/score_docs_as_code+/src/tests/testfile.py
-    => score_docs_as_code
+    Parse out the Module-Name from the filename:
+    Combo Example:
+        Path: external/score_docs_as_code+/src/helper_lib/test_helper_lib.py
+        => score_docs_as_code
+    Local:
+        Path: src/helper_lib/test_helper_lib.py
+        => local_module
+
     """
 
     # COMBO BUILD
-    # If external is in the filepath that gets parsed =>
-    # file is in an external module => combo build
-    # Example Path:
-    #   PosixPath('external/score_docs_as_code+/src/helper_lib/test_helper_lib.py'
 
     if str(path).startswith("external/"):
         # This allows for files / folders etc. to have `external` in their name too.
         module_raw = str(path).removeprefix("external/")
         filepath_split = str(module_raw).split("/", maxsplit=1)
         return str(filepath_split[0].removesuffix("+"))
     # We return this when we are in a local build `//:docs` the rest of DaC knows
-    # What to do then if it encounters this module_name
     return "local_module"
 
 

src/extensions/score_source_code_linker/module_source_links.py

@@ -78,7 +78,7 @@ def store_module_source_links_json(
     # After `rm -rf _build` or on clean builds the directory does not exist, so we need
     # to create it
     file.parent.mkdir(exist_ok=True)
-    with open(file, "w") as f:
+    with open(file, "w", encoding="utf-8") as f:
         json.dump(
             source_code_links,
             f,

src/extensions/score_source_code_linker/need_source_links.py

@@ -123,9 +123,6 @@ def group_by_need(
 
       {
         "need": "<need_id>",
-        "module_name": <module_name>,
-        "hash": <git hash>,
-        "url": <github base url>,
         "links": {
           "CodeLinks": [NeedLink, NeedLink, ...],
           "TestLinks": [testlink, testlink, ...]

src/incremental.py

@@ -90,6 +90,8 @@ def get_env(name: str) -> str:
         base_arguments.append(f"-A=github_repo={args.github_repo}")
         base_arguments.append("-A=github_version=main")
         base_arguments.append(f"-A=doc_path={get_env('SOURCE_DIRECTORY')}")
+    if os.getenv("KNOWN_GOOD_JSON"):
+        base_arguments.append(f"--define=KNOWN_GOOD_JSON={get_env('KNOWN_GOOD_JSON')}")
 
     action = get_env("ACTION")
     if action == "live_preview":