From ae4739271e3fdc98219beed04a719666bde6f3e5 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 11 Jun 2025 17:13:52 +0200 Subject: [PATCH 01/54] migrated src tracing here and add infra --- .gitignore | 17 + .pre-commit-config.yaml | 27 + pyproject.toml | 170 ++++++ src/sphinx_codelinks/README.md | 45 ++ src/sphinx_codelinks/__init__.py | 10 + src/sphinx_codelinks/cmd.py | 179 +++++++ src/sphinx_codelinks/source_discover.py | 69 +++ .../sphinx_extension/config.py | 144 +++++ .../sphinx_extension/debug.py | 189 +++++++ .../sphinx_extension/directives/src_trace.py | 339 ++++++++++++ .../sphinx_extension/html_wrapper.py | 52 ++ .../sphinx_extension/source_tracing.py | 197 +++++++ .../sphinx_extension/ub_sct.css | 8 + src/sphinx_codelinks/virtual_docs/config.py | 204 +++++++ .../virtual_docs/ubt_models.py | 125 +++++ src/sphinx_codelinks/virtual_docs/utils.py | 215 ++++++++ .../virtual_docs/virtual_docs.py | 241 +++++++++ tests/__init__.py | 0 ...[sphinx_project0-source_code0].doctree.xml | 27 + ...[sphinx_project1-source_code1].doctree.xml | 3 + tests/conftest.py | 63 +++ tests/data/dcdc/charge/demo_1.cpp | 28 + tests/data/dcdc/charge/demo_2.cpp | 48 ++ tests/data/dcdc/discharge/demo_3.cpp | 56 ++ tests/data/dcdc/supercharge.cpp | 31 ++ .../oneline_comment_basic/basic_oneliners.c | 29 + .../oneline_comment_basic/vdoc_config.toml | 19 + .../default_oneliners.c | 19 + .../oneline_comment_default/vdoc_config.toml | 5 + tests/data/sphinx/Makefile | 20 + tests/data/sphinx/conf.py | 113 ++++ tests/data/sphinx/index.rst | 11 + tests/data/sphinx/make.bat | 35 ++ tests/data/sphinx/src_trace.toml | 28 + tests/doc_test/recursive_dirs/conf.py | 66 +++ .../recursive_dirs/dummy_src_lv1/dummy_1.cpp | 8 + .../dummy_src_lv1/dummy_lv2/dummy_2.cpp | 8 + .../dummy_lv2/dummy_lv3/dummy_3.cpp | 8 + .../dummy_lv2/dummy_lv3/dummy_lv4/dummy_4.cpp | 8 + tests/doc_test/recursive_dirs/index.rst | 2 + tests/doc_test/recursive_dirs/src_trace.toml | 26 + tests/test_cmd.py | 144 +++++ tests/test_source_discover.py | 38 ++ tests/test_src_trace.py | 48 ++ tests/test_virtual_docs.py | 497 ++++++++++++++++++ 45 files changed, 3619 insertions(+) create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml create mode 100644 src/sphinx_codelinks/README.md create mode 100644 src/sphinx_codelinks/__init__.py create mode 100644 src/sphinx_codelinks/cmd.py create mode 100644 src/sphinx_codelinks/source_discover.py create mode 100644 src/sphinx_codelinks/sphinx_extension/config.py create mode 100644 src/sphinx_codelinks/sphinx_extension/debug.py create mode 100644 src/sphinx_codelinks/sphinx_extension/directives/src_trace.py create mode 100644 src/sphinx_codelinks/sphinx_extension/html_wrapper.py create mode 100644 src/sphinx_codelinks/sphinx_extension/source_tracing.py create mode 100644 src/sphinx_codelinks/sphinx_extension/ub_sct.css create mode 100644 src/sphinx_codelinks/virtual_docs/config.py create mode 100644 src/sphinx_codelinks/virtual_docs/ubt_models.py create mode 100644 src/sphinx_codelinks/virtual_docs/utils.py create mode 100644 src/sphinx_codelinks/virtual_docs/virtual_docs.py create mode 100644 tests/__init__.py create mode 100644 tests/__snapshots__/test_src_trace/test_build_html[sphinx_project0-source_code0].doctree.xml create mode 100644 tests/__snapshots__/test_src_trace/test_build_html[sphinx_project1-source_code1].doctree.xml create mode 100644 tests/conftest.py create mode 100644 tests/data/dcdc/charge/demo_1.cpp create mode 100644 tests/data/dcdc/charge/demo_2.cpp create mode 100644 tests/data/dcdc/discharge/demo_3.cpp create mode 100644 tests/data/dcdc/supercharge.cpp create mode 100644 tests/data/oneline_comment_basic/basic_oneliners.c create mode 100644 tests/data/oneline_comment_basic/vdoc_config.toml create mode 100644 tests/data/oneline_comment_default/default_oneliners.c create mode 100644 tests/data/oneline_comment_default/vdoc_config.toml create mode 100644 tests/data/sphinx/Makefile create mode 100644 tests/data/sphinx/conf.py create mode 100644 tests/data/sphinx/index.rst create mode 100644 tests/data/sphinx/make.bat create mode 100644 tests/data/sphinx/src_trace.toml create mode 100644 tests/doc_test/recursive_dirs/conf.py create mode 100644 tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_1.cpp create mode 100644 tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_2.cpp create mode 100644 tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_3.cpp create mode 100644 tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_lv4/dummy_4.cpp create mode 100644 tests/doc_test/recursive_dirs/index.rst create mode 100644 tests/doc_test/recursive_dirs/src_trace.toml create mode 100644 tests/test_cmd.py create mode 100644 tests/test_source_discover.py create mode 100644 tests/test_src_trace.py create mode 100644 tests/test_virtual_docs.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fcd7543 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# python generated files +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info + +# venv +.venv + +# lock files +requirements.lock +requirements-dev.lock + +# Sphinx build output +**/_build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..da4d0bb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: pretty-format-json + args: [--autofix, --no-sort-keys] + files: (package\.json|tsconfig\.json)$ + types: [file] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.1 + hooks: + - id: ruff-format + name: python format + - id: ruff + alias: ruff-check + name: python lint + args: [--fix] + + - repo: https://github.com/ComPWA/taplo-pre-commit + rev: v0.9.3 + hooks: + - id: taplo-format + # lint fetches schemas online at each call, deactivate for now + - id: taplo-lint diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5aa5db8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,170 @@ +[project] +name = "sphinx-codelinks" +version = "0.1.0" +description = "Add your description here" +authors = [{ name = "team useblocks", email = "info@useblocks.com" }] +maintainers = [ + { name = "Marco Heinemann", email = "marco.heinemann@useblocks.com" }, +] +license = { file = "LICENSE" } +readme = "README.md" +requires-python = ">= 3.12" +dependencies = [ + "comment-parser>=1.2.4", + "gitignore-parser>=0.1.11", + "typer>=0.16.0", + "jsonschema", + "sphinx>=7.4,<9", + # unconstrained versions, to be pinned by user or Sphinx + "jinja2", + "pygments", + "docutils", # constrained by user or Sphinx +] + +[build-system] +requires = ["flit_core >=3.4,<4"] +build-backend = "flit_core.buildapi" + +[tool.rye] +managed = true +dev-dependencies = [ + "types-docutils", + "types-Pygments", + "syrupy>=4.9.1", + "furo>=2024.5.6", + "moto ~= 5.0", + "mypy>=1.12.1", + "myst-parser>=4.0.0", + "pip-licenses>=5.0.0", + "psutil>=7.0.0", + "pytest-cov>=5.0.0", + "pytest>=8.2.2", + "simple-build>=0.0.2", + "sphinx-design>=0.5.0", + "sphinx-needs>=4.2.0", + "types-psutil>=7.0.0.20250218", + "uv>=0.5.5", + "pytest-docker>=3.1.2", + "shiv>=1.0.8", + "insta-science>=0.2.1", + "types-jsonschema>=4.23.0.20241208", +] + +[tool.ruff.lint] +extend-select = [ + # "ANN", + "S", + "ARG", + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "FURB", # refurb (modernising code) + "I", # isort + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "N", # pep8-naming + "PERF", # perflint (performance anti-patterns) + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PTH", # flake8-use-pathlib + "RUF", # Ruff-specific rules + "SIM", # flake8-simplify + "SLF", # private member access + "UP", # pyupgrade + "T20", # flake8-print +] +extend-ignore = [ + "ISC001", # implicit-str-concat +] + +[tool.ruff.lint.isort] +split-on-trailing-comma = false +force-sort-within-sections = true + +[tool.ruff.lint.per-file-ignores] +"**/tests/*" = [ + "ARG001", # unused-function-argument - fixtures + "ARG005", # unused-lambda-argument - monkeypatches + "PLR2004", # magic-value-comparison - valueable for tests + "S101", # assert - needed for tests +] +"**/build_hooks_*/**" = [ + "S607", # start-process-with-partial-path - pyarmor call in rye context + "S603", # subprocess-without-shell-equals-true - pyarmor call +] +"scripts/*.py" = [ + "T201", # print - used for output + "S607", # start-process-with-partial-path - pyarmor call in rye context + "S603", # subprocess-without-shell-equals-true - build scripts +] +"src/sphinx_codelinks/sphinx_extension/debug.py" = [ + "T201", # print - used for output +] + +[tool.mypy] +exclude = [ + "^.*/tests/", + "dist/", + "docs/_build/", + "docs/conf.py", + "python/ubt_autodiscover/src/", # TODO still untyped + "_python_archive/ubt_connect_core/src/", # TODO still untyped + "_python_archive/ubt_connect_services/src/", # TODO still untyped + "python/ubt_db/src/", # TODO still untyped + "python/ubt_runtime/src/ubt_runtime/__init__.py", # generated source + "python/ubt_server/src/", # TODO still untyped + "python/ubt_sphinx/src/", # TODO still untyped + "scripts/sync_metadata.py", # TODO investigate why this fails here but not in ubcode +] +show_error_codes = true +warn_unused_ignores = true +warn_redundant_casts = true +strict = true +# disallow dynamic typing +disallow_any_unimported = true +disallow_any_expr = true +# disallow_any_decorated = true +disallow_any_explicit = true +disallow_any_generics = true +disallow_subclassing_any = true +# dissallow untyped definitions and calls +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_decorators = true + +plugins = ["pydantic.mypy"] +mypy_path = "typings" + +[[tool.mypy.overrides]] +module = ["licensing.*", "tomlkit.*"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = ["scripts.*", "build_hooks.*", "ublicense.*"] +disallow_any_expr = false + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_any_explicit = false +disallow_any_unimported = false +disallow_untyped_defs = false +disallow_any_expr = false + +[[tool.mypy.overrides]] +module = ["ublicense.*"] +disallow_any_expr = false + +[[tool.mypy.overrides]] +module = "ubt_source_tracing.*" +disallow_any_unimported = false +disallow_untyped_defs = false +disallow_any_expr = false + +[tool.pydantic-mypy] +init_forbid_extra = true +init_typed = true +warn_required_dynamic_aliases = true + +[tool.pytest.ini_options] +testpaths = ["python"] diff --git a/src/sphinx_codelinks/README.md b/src/sphinx_codelinks/README.md new file mode 100644 index 0000000..771eb37 --- /dev/null +++ b/src/sphinx_codelinks/README.md @@ -0,0 +1,45 @@ +# CodeLinks + +## Overview + +This is a Sphinx extension to extract Sphinx-Needs items from source files +such as C, C++ and others. + +The need items are defined in the source files as comments and can be used to define +test case specifications or implementation markers. + +Various definition styles are supported, such as one-line, multi-line or raw RST. + +The project consists of the following three components: + +- Source Discovery: determines list of source files from a given directory +- Virtual Docs: extract need annotations while keeping the source map +- Source Tracing: Sphinx extension to represent the collected the needs in the documentation + +`Source Discovery` and `Virtual Docs` can be used as `APIs` or `CLI tools`. +The detail usages can be found in the [test cases](./python/ubt_source_tracing/tests). + +The library is built to be +- ⚡ fast for large code bases and +- 📃 support a multitude of languages. + +## Source Discovery + +Recursively collect the file paths from a given directory. +It can be configured to respect `.gitignore`. + +## Virtual Docs + +Virtual Docs parses the discoverd files and + +- extracts the need items from the comments in the source files. +- extracts additional metadata such as extra options and links. +- generates virtual documents containing the above-mentioned information into `json` files. +- caches virtual docs for incremental builds. +- keeps the source map to the path and line number of the original source files. + +## CodeLinks + +CodeLinks is a Sphinx Extension based on Sphinx-Needs. It provides the directive `src-tracing` +to collect the needs defined in source files by using `Source Discovery` and `Virtual Docs` +under the hood. diff --git a/src/sphinx_codelinks/__init__.py b/src/sphinx_codelinks/__init__.py new file mode 100644 index 0000000..8118e2a --- /dev/null +++ b/src/sphinx_codelinks/__init__.py @@ -0,0 +1,10 @@ +"""ubTrace source code analyzer""" + +from ubt_source_tracing.sphinx_extension.source_tracing import setup + +__version__ = "0.1.0" + +__all__ = [ + "__version__", + "setup", +] diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py new file mode 100644 index 0000000..4700813 --- /dev/null +++ b/src/sphinx_codelinks/cmd.py @@ -0,0 +1,179 @@ +from os import linesep +from pathlib import Path +import tempfile +import tomllib +from typing import Annotated, Any + +import typer + +app = typer.Typer( + no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]} +) + + +@app.command(no_args_is_help=True) +def discover( + root_dir: Annotated[ + Path, + typer.Argument( + ..., + help="Root directory for discovery", + show_default=False, + dir_okay=True, + file_okay=False, + exists=True, + resolve_path=True, + ), + ], + excludes: Annotated[ + list[str] | None, + typer.Option( + "--excludes", + "-e", + help="Glob patterns to be excluded.", + ), + ] = None, + includes: Annotated[ + list[str] | None, + typer.Option( + "--includes", + "-i", + help="Glob patterns to be included.", + ), + ] = None, + gitignore: Annotated[bool, typer.Option(help="Respect .gitignore(s)")] = True, + file_types: Annotated[ + list[str] | None, + typer.Option( + "--file-type", + "-f", + help="The file extension to be discovered. If not specified, all files are discovered.", + ), + ] = None, +) -> None: + """Discover the filepaths from the given root directory.""" + from ubt_source_tracing.source_discover import SourceDiscover + + source_discover = SourceDiscover( + root_dir=root_dir, + excludes=excludes, + includes=includes, + file_types=file_types, + gitignore=gitignore, + ) + typer.echo(f"{len(source_discover.source_paths)} files discovered") + for file_path in source_discover.source_paths: + typer.echo(file_path) + + +@app.command(no_args_is_help=True) +def vdoc( + config: Annotated[ + Path, + typer.Option( + "--config", + "-c", + help="The toml config file", + show_default=False, + dir_okay=False, + file_okay=True, + exists=True, + ), + ], + project: Annotated[ + str | None, typer.Option("--project", "-p", help="project identifier in config") + ] = None, + output_dir: Path = typer.Option( # noqa: B008 # to support filepath + Path(tempfile.gettempdir()), # noqa: B008 # to support filepath + "--output-dir", + "-o", + help="The output directory of generated documents and caches.", + ), +) -> None: + """Generate virtual documents for caching and extract the oneline comments.""" + + from ubt_source_tracing.virtual_docs.config import OneLineCommentStyle + + data = load_config_from_toml(config, project) + # src_dir = Path(data["src_dir"]) + # # if not project: + # # src_tracing extension assume root_dir as the config + root_dir = config.parent + src_dir = (root_dir / Path(data["src_dir"])).resolve() + + comment_type = data["comment_type"] + gitignore = data["gitignore"] + excludes = data["exclude"] + includes = data["include"] + oneline_comment_style = data.get("oneline_comment_style") + if oneline_comment_style is None: + oneline_comment_style = OneLineCommentStyle() + else: + oneline_comment_style = OneLineCommentStyle(**oneline_comment_style) + + errors = oneline_comment_style.check_fields_configuration() + if errors: + raise typer.BadParameter( + f"Invalid oneline comment style configuration: {linesep.join(errors)}" + ) + from ubt_source_tracing.source_discover import SourceDiscover + from ubt_source_tracing.virtual_docs.utils import get_file_types + + file_types = get_file_types(comment_type) + + source_discover = SourceDiscover( + root_dir=src_dir, + excludes=excludes, + includes=includes, + file_types=file_types, + gitignore=gitignore, + ) + + from ubt_source_tracing.virtual_docs.virtual_docs import VirtualDocs + + virtual_docs = VirtualDocs( + src_files=source_discover.source_paths, + src_dir=str(src_dir), + output_dir=str(output_dir), + oneline_comment_style=oneline_comment_style, + comment_type=comment_type, + ) + virtual_docs.collect() + virtual_docs.dump_virtual_docs() + + if len(virtual_docs.virtual_docs) > 0: + typer.echo("The virtual documents are generated:") + for v_doc in virtual_docs.virtual_docs: + json_path = output_dir / v_doc.filepath.with_suffix(".json").relative_to( + src_dir + ) + typer.echo(json_path) + else: + typer.echo("No virtual documents are generated.") + + virtual_docs.cache.update_cache() + typer.echo("The cached files are:") + for cached_file in virtual_docs.cache.cached_files: + typer.echo(cached_file) + + +def load_config_from_toml( # type: ignore[misc] + toml_file: Path, project: str | None = None +) -> dict[str, Any]: + try: + with toml_file.open("rb") as f: + toml_data = tomllib.load(f) + + if project: + toml_data = toml_data["src_trace"]["projects"][project] + + except Exception as e: + raise Exception( + f"Failed to load source tracing configuration from {toml_file}" + ) from e + + return toml_data + + +if __name__ == "__main__": + app() diff --git a/src/sphinx_codelinks/source_discover.py b/src/sphinx_codelinks/source_discover.py new file mode 100644 index 0000000..badae11 --- /dev/null +++ b/src/sphinx_codelinks/source_discover.py @@ -0,0 +1,69 @@ +from collections.abc import Callable +import fnmatch +import os +from pathlib import Path + +from gitignore_parser import parse_gitignore # type: ignore[import-untyped] + + +class SourceDiscover: + def __init__( + self, + root_dir: Path, + excludes: list[str] | None = None, + includes: list[str] | None = None, + gitignore: bool = True, + file_types: list[str] | None = None, + ): + self.root_path = root_dir + self.excludes = excludes + self.includes = includes + # Only gitignore at source root is considered. + # TODO: Support nested gitignore files + gitignore_path = self.root_path / ".gitignore" + self.gitignore_matcher: Callable[[str], bool] | None = ( + parse_gitignore(gitignore_path) + if gitignore and gitignore_path.exists() + else None + ) + # normalize the file types to lower case with leading dot + self.file_types = ( + { + file_type.lower() + if file_type.startswith(".") + else f".{file_type}".lower() + for file_type in file_types + } + if file_types + else None + ) + + self.source_paths = self._discover() + + def _discover(self) -> list[Path]: + """Discover source files recursively in the given directory.""" + discovered_files = [] + for filepath in self.root_path.rglob("*"): + if filepath.is_file(): + if self.file_types and filepath.suffix.lower() not in self.file_types: + continue + rel_filepath = str(filepath.relative_to(self.root_path)) + if self.includes and self._matches_any(rel_filepath, self.includes): + # "includes" has the highest priority over "gitignore" and "excludes" + discovered_files.append(filepath) + continue + if self.gitignore_matcher and self.gitignore_matcher( + str(filepath.absolute()) + ): + continue + if self.excludes and self._matches_any(rel_filepath, self.excludes): + continue + discovered_files.append(filepath) + sorted_filepaths = sorted( + discovered_files, key=lambda x: os.path.normcase(os.path.normpath(x)) + ) + return sorted_filepaths + + def _matches_any(self, rel_filepath: str, patterns: list[str]) -> bool: + """Check if the given file path matches any of the given patterns.""" + return any(fnmatch.fnmatch(rel_filepath, pattern) for pattern in patterns) diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py new file mode 100644 index 0000000..00e0549 --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -0,0 +1,144 @@ +from dataclasses import MISSING, dataclass, field, fields +from typing import Any, Literal, TypedDict, cast + +from sphinx.application import Sphinx +from sphinx.config import Config as _SphinxConfig +from ubt_source_tracing.virtual_docs.config import ( + OneLineCommentStyle, + OneLineCommentStyleType, +) + +SRC_TRACE_CACHE: str = "src_trace_cache" + + +class SourceTracingLineHref: + """Global class for the mapping between source file line numbers and Sphinx documentation links.""" + + def __init__(self) -> None: + self.mappings: dict[str, dict[int, str]] = {} + + +file_lineno_href = SourceTracingLineHref() + + +class SrcTraceProjectConfigType(TypedDict): + # only support C/C++ for now + comment_type: Literal["cpp", "hpp", "c", "h"] + src_dir: str + remote_url_pattern: str + exclude: list[str] + include: list[str] + gitignore: bool + oneline_comment_style: OneLineCommentStyle + + +@dataclass +class SrcTraceSphinxConfig: + def __init__(self, config: _SphinxConfig) -> None: + super().__setattr__("_config", config) + + def __getattribute__(self, name: str) -> Any: # type: ignore[misc] + if name.startswith("__") or name == "_config": + return super().__getattribute__(name) + return getattr(super().__getattribute__("_config"), f"src_trace_{name}") + + def __setattr__(self, name: str, value: Any) -> None: # type: ignore[misc] + if name == "_config" and "src_trace_projects" in value: + src_trace_projects: dict[str, SrcTraceProjectConfigType] = value[ + "src_trace_projects" + ] + for _config in src_trace_projects.values(): + # overwrite the config into different types on purpose + # covert dict to OneLineCommentStyle class + oneline_comment_style: OneLineCommentStyleType | None = cast( + OneLineCommentStyleType, _config.get("oneline_comment_style") + ) + if not oneline_comment_style: + raise Exception("OneLineCommentStyle is not given") + + _config["oneline_comment_style"] = OneLineCommentStyle( + **oneline_comment_style + ) + if name.startswith("__") or name == "_config": + return super().__setattr__(name, value) + + return setattr(super().__getattribute__("_config"), f"src_trace_{name}", value) + + @classmethod + def add_config_values(cls, app: Sphinx) -> None: + """Add all config values to Sphinx application""" + for item in fields(cls): + if item.default_factory is not MISSING: + default = item.default_factory() + elif item.default is not MISSING: + default = item.default + else: + raise Exception(f"Field {item.name} has no default value or factory") + + name = item.name + app.add_config_value( + f"src_trace_{name}", + default, + item.metadata["rebuild"], + types=item.metadata["types"], + ) + + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + config_from_toml: str | None = field( + default=None, + metadata={ + "rebuild": "env", + "types": (str, type(None)), + "schema": { + "type": ["string", "null"], + "examples": ["config.toml", None], + }, + }, + ) + """Path to a TOML file to load configuration from.""" + + set_local_url: bool = field( + default=False, + metadata={ + "rebuild": "env", + "types": (bool,), + }, + ) + """Set the file URL in the extracted need.""" + + local_url_field: str = field( + default="local-url", + metadata={"rebuild": "env", "types": (str,)}, + ) + """The field name for the file URL in the extracted need.""" + + set_remote_url: bool = field( + default=False, + metadata={ + "rebuild": "env", + "types": (bool,), + }, + ) + remote_url_field: str = field( + default="remote-url", + metadata={"rebuild": "env", "types": (str,)}, + ) + """The field name for the remote URL in the extracted need.""" + + projects: dict[str, SrcTraceProjectConfigType] = field( + default_factory=dict, + metadata={"rebuild": "env", "types": ()}, + ) + """The configuration for the source tracing projects.""" + + debug_measurement: bool = field( + default=False, metadata={"rebuild": "html", "types": (bool,)} + ) + """If True, log runtime information for various functions.""" + debug_filters: bool = field( + default=False, metadata={"rebuild": "html", "types": (bool,)} + ) + """If True, log filter processing runtime information.""" diff --git a/src/sphinx_codelinks/sphinx_extension/debug.py b/src/sphinx_codelinks/sphinx_extension/debug.py new file mode 100644 index 0000000..28a8078 --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/debug.py @@ -0,0 +1,189 @@ +""" +Contains debug features to track down +runtime and other problems with Src-Trace +""" + +from __future__ import annotations + +from collections.abc import Callable +from datetime import datetime +from functools import wraps +import inspect +import json +from pathlib import Path +from timeit import default_timer as timer # Used for timing measurements +from typing import Any, TypeVar + +from jinja2 import Environment, PackageLoader, select_autoescape +from sphinx.application import Sphinx + +# Stores the timing results +TIME_MEASUREMENTS: dict[str, Any] = {} # type: ignore[misc] +EXECUTE_TIME_MEASUREMENTS = ( + False # Will be used to de/activate measurements. Set during a Sphinx Event +) + +START_TIME = 0.0 + +T = TypeVar("T", bound=Callable[..., Any]) # type: ignore[misc] + + +def measure_time( # type: ignore[misc] + category: str | None = None, source: str = "internal", name: str | None = None +) -> Callable[[T], T]: + """ + Decorator for measuring the needed execution time of a specific function. + + It measures: + + * Amount of executions + * Overall time consumed + * Average time of an execution as `avg` + * Minimum time of an execution as `min` + * Maximum time of an execution as `max` + + For `max` also the used function parameters are stored as string values, to make + it easier to reproduce the maximum case. + + Usage as decorator:: + + from sphinx_needs.utils import measure_time + + @measure_time('my_category') + def my_cool_function(a, b,c ): + # does something + + :param category: Name of a category, which helps to cluster the measured functions. + :param source: Should be "internal" or "user". Used to easily structure function written by user. + :param name: Name to use for the measured. If not given, the function name is used. + """ + + def inner(func: T) -> T: # type: ignore[misc] + @wraps(func) + def wrapper(*args: list[object], **kwargs: dict[object, object]) -> Any: # type: ignore[misc] + """ + Wrapper function around a given/decorated function, which cares about measurement and storing the result + + :param args: Arguments for the original function + :param kwargs: Keyword arguments for the original function + """ + if not EXECUTE_TIME_MEASUREMENTS: + return func(*args, **kwargs) + + start = timer() + # Execute original function + result = func(*args, **kwargs) + end = timer() + + runtime = end - start + + mt_name = func.__name__ if name is None else name + + mt_id = f"{category}_{func.__name__}" + + if mt_id not in TIME_MEASUREMENTS: + TIME_MEASUREMENTS[mt_id] = { + "name": mt_name, + "category": category, + "source": source, + "doc": func.__doc__, + "file": inspect.getfile(func), + "line": inspect.getsourcelines(func)[1], + "amount": 0, + "overall": 0, + "avg": None, + "min": None, + "max": None, + "min_max_spread": None, + "max_params": {"args": [], "kwargs": {}}, + } + + runtime_dict = TIME_MEASUREMENTS[mt_id] + + runtime_dict["amount"] += 1 + runtime_dict["overall"] += runtime + + if runtime_dict["min"] is None or runtime < runtime_dict["min"]: + runtime_dict["min"] = runtime + + if runtime_dict["max"] is None or runtime > runtime_dict["max"]: + runtime_dict["max"] = runtime + runtime_dict["max_params"] = { # Store parameters as a shorten string + "args": str([str(arg)[:80] for arg in args]), + "kwargs": str( + {key: str(value)[:80] for key, value in kwargs.items()} + ), + } + runtime_dict["min_max_spread"] = ( + runtime_dict["max"] / runtime_dict["min"] * 100 + ) + runtime_dict["avg"] = runtime_dict["overall"] / runtime_dict["amount"] + return result + + return wrapper # type: ignore[return-value] + + return inner + + +def measure_time_func( # type: ignore[misc] + func: T, + category: str | None = None, + source: str = "internal", + name: str | None = None, +) -> T: + """Wrapper for measuring the needed execution time of a specific function. + + Usage as function:: + + from sphinx_needs.utils import measure_time + + # Old call: my_cool_function(a,b,c) + new_func = measure_time_func('my_category', func=my_cool_function) + new_func(a,b,c) + """ + return measure_time(category, source, name)(func) + + +def _print_timing_results() -> None: + for value in TIME_MEASUREMENTS.values(): + print(value["name"]) + print(f" amount: {value['amount']}") + print(f" overall: {value['overall']:2f}") + print(f" avg: {value['avg']:2f}") + print(f" max: {value['max']:2f}") + print(f" min: {value['min']:2f} \n") + + +def _store_timing_results_json(app: Sphinx, build_data: dict[str, Any]) -> None: # type: ignore[misc] + json_result_path = Path(app.outdir) / "debug_measurement.json" + + data = {"build": build_data, "measurements": TIME_MEASUREMENTS} + with json_result_path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=4) + print(f"Timing measurement results (JSON) stored under {json_result_path}") + + +def _store_timing_results_html(app: Sphinx, build_data: dict[str, Any]) -> None: # type: ignore[misc] + jinja_env = Environment( + loader=PackageLoader("sphinx_needs"), autoescape=select_autoescape() + ) + template = jinja_env.get_template("time_measurements.html") + out_file = Path(str(app.outdir)) / "debug_measurement.html" + with out_file.open("w", encoding="utf-8") as f: + f.write(template.render(data=TIME_MEASUREMENTS, build_data=build_data)) + print(f"Timing measurement report (HTML) stored under {out_file}") + + +def process_timing(app: Sphinx, _exception: Exception | None) -> None: + if EXECUTE_TIME_MEASUREMENTS: + build_data = { + "project": app.config["project"], + "start": START_TIME, + "end": timer(), + "duration": timer() - START_TIME, + "timestamp": datetime.now().isoformat(), + } + + _print_timing_results() + _store_timing_results_json(app, build_data) + _store_timing_results_html(app, build_data) diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py new file mode 100644 index 0000000..977ebee --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -0,0 +1,339 @@ +from collections.abc import Callable +from pathlib import Path +import subprocess +from typing import Any, ClassVar, cast + +from docutils import nodes +from docutils.parsers.rst import directives +from packaging.version import Version +import sphinx +from sphinx.util.docutils import SphinxDirective +from sphinx_needs.api import add_need # type: ignore[import-untyped] +from sphinx_needs.utils import add_doc # type: ignore[import-untyped] +from ubt_source_tracing.source_discover import SourceDiscover +from ubt_source_tracing.sphinx_extension.config import ( + SRC_TRACE_CACHE, + SrcTraceProjectConfigType, + SrcTraceSphinxConfig, + file_lineno_href, +) +from ubt_source_tracing.sphinx_extension.debug import measure_time +from ubt_source_tracing.virtual_docs.ubt_models import UBTComment +from ubt_source_tracing.virtual_docs.utils import get_file_types +from ubt_source_tracing.virtual_docs.virtual_docs import VirtualDocs + +sphinx_version = sphinx.__version__ + + +if Version(sphinx_version) >= Version("1.6"): + from sphinx.util import logging +else: + import logging # type: ignore[no-redef] + +logger = logging.getLogger(__name__) + + +def generate_str_link_name( + comment: UBTComment, target_filepath: Path, target_dir: str +) -> str: + if comment.start_line == comment.end_line: + lineno = f"L{comment.start_line}" + else: + lineno = f"L{comment.start_line}-L{comment.end_line}" + url = str(target_filepath.relative_to(target_dir)) + f"#{lineno}" + + return url + + +def get_git_commit_id(src_dir: Path) -> str: + try: + commit_id = ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=src_dir) # noqa: S607, S603 + .decode("utf-8") + .strip() + ) + except subprocess.CalledProcessError as err: + # raise RuntimeError("Failed to get the latest commit ID") from err + logger.warning(f"Failed to get the latest commit ID: {err}") + commit_id = "" + return commit_id + + +def get_git_root(cwd: Path = Path()) -> Path | None: + try: + # Run the git command to get the root directory + git_root = subprocess.check_output( # noqa: S603 + ["git", "rev-parse", "--show-toplevel"], # noqa: S607 + cwd=cwd, + text=True, # Ensures the output is a string + ).strip() + return Path(git_root) + except subprocess.CalledProcessError: + logger.warning(f"Failed to get the Git root directory for {cwd}.") + return None + + +def validate_option(options: dict[str, str]) -> None: + if "project" not in options: + raise ValueError("Project option must be set.") + if "file" in options and "directory" in options: + raise ValueError("Either file or directory options can be set.") + + +class SourceTracing(nodes.General, nodes.Element): + pass + + +class SourceTracingDirective(SphinxDirective): + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + # this enables content in the directive + has_content = True + option_spec: ClassVar[dict[str, Callable[[str], str]] | None] = { + "id": directives.unchanged_required, + "project": directives.unchanged_required, + "file": directives.unchanged_required, + "directory": directives.unchanged_required, + } + + @measure_time("src-trace") + def run(self) -> list[nodes.Node]: + validate_option(self.options) + + project = self.options["project"] + title = self.arguments[0] + # get source tracing config + src_trace_sphinx_config = SrcTraceSphinxConfig(self.env.config) + + # load config + src_trace_conf: SrcTraceProjectConfigType = src_trace_sphinx_config.projects[ + project + ] + comment_type = src_trace_conf["comment_type"] + oneline_comment_style = src_trace_conf["oneline_comment_style"] + + src_dir = self.locate_src_dir(src_trace_sphinx_config, src_trace_conf) + + out_dir = Path(self.env.app.outdir) + # the directory where the source files are copied to + target_dir = out_dir / src_dir.name + + extra_options = {"project": project} + source_files = self.get_src_files(self.options, src_dir, src_trace_conf) + + # add source files into the dependency + # https://www.sphinx-doc.org/en/master/extdev/envapi.html#sphinx.environment.BuildEnvironment.note_dependency + for source_file in source_files: + self.env.note_dependency(str(source_file.resolve())) + + virtual_docs = VirtualDocs( + source_files, + str(src_dir), + str(out_dir / SRC_TRACE_CACHE), + oneline_comment_style, + comment_type=comment_type, + ) + virtual_docs.collect() + + needs = [] + + # create the need for src-trace directive + src_trace_need = add_need( + app=self.env.app, # The Sphinx application object + state=self.state, # The docutils state object + docname=self.env.docname, # The current document name + lineno=self.lineno, # The line number where the directive is used + need_type="srctrace", # The type of the need + title=title, # The title of the need + **extra_options, + ) + needs.extend(src_trace_need) + + # inject needs_string_links config before add_need() + # https://sphinx-needs.readthedocs.io/en/latest/configuration.html#needs-string-links + # local URL + local_url_field = None + remote_url_field = None + if src_trace_sphinx_config.set_local_url: + local_url_field = src_trace_sphinx_config.local_url_field + self.env.config.needs_string_links[local_url_field] = { + "regex": r"^(?P.+?)\.[^\.]+#L(?P\d+)", + "link_url": ( + f"file://{target_dir!s}/{{{{value}}}}.html#L-{{{{lineno}}}}" + ), + "link_name": "{{value}}#L{{lineno}}", + "options": [local_url_field], + } + if ( + src_trace_sphinx_config.set_remote_url + and src_trace_conf["remote_url_pattern"] + ): + git_root_path: Path | None = get_git_root(src_dir) + remote_url_field = src_trace_sphinx_config.remote_url_field + commit_id = get_git_commit_id(src_dir) + if git_root_path is None: + # No git root found, use the source directory as the remote source directory + remote_src_dir = src_dir + else: + remote_src_dir = src_dir.relative_to(git_root_path) + remote_url_pattern = src_trace_conf["remote_url_pattern"].format( + commit=commit_id, + path=f"{remote_src_dir}/" + "{{value}}", + line="{{lineno}}", + ) + self.env.config.needs_string_links[remote_url_field] = { + "regex": r"^(?P.+)#L(?P.*)?", + "link_url": remote_url_pattern, + "link_name": "{{value}}#L{{lineno}}", + "options": [remote_url_field], + } + dirs = { + "src_dir": src_dir, + "out_dir": out_dir, + "target_dir": target_dir, + } + # render needs from the source files + rendered_needs = self.render_needs( + virtual_docs, + local_url_field, + remote_url_field, + dirs, + ) + if rendered_needs: + needs.extend(rendered_needs) + + # virtual docs caching + virtual_docs.dump_virtual_docs() + virtual_docs.cache.update_cache() + + # for post-processing of need links + # https://github.com/useblocks/sphinx-needs/issues/1210 + add_doc(self.env, self.env.docname) + + return needs + + def get_src_files( + self, + extra_options: dict[str, str], + src_dir: Path, + src_trace_conf: SrcTraceProjectConfigType, + ) -> list[Path]: + source_files = [] + if "file" in self.options: + file: str = self.options["file"] + filepath = src_dir / file + source_files.append(filepath.resolve()) + extra_options["file"] = file + else: + directory = self.options.get("directory") + if directory is None: + # when neither "file" and "directory" are given, the project root dir is by default + directory = "./" + else: + extra_options["directory"] = directory + dir_path = src_dir / directory + file_types = get_file_types(src_trace_conf["comment_type"]) + source_discover = SourceDiscover( + dir_path, + gitignore=src_trace_conf["gitignore"], + includes=src_trace_conf["include"], + excludes=src_trace_conf["exclude"], + file_types=file_types, + ) + source_files.extend(source_discover.source_paths) + + return source_files + + def locate_src_dir( + self, + src_trace_sphinx_config: SrcTraceSphinxConfig, + src_trace_conf: SrcTraceProjectConfigType, + ) -> Path: + """Locate the source directory based on the configuration.""" + # src dir in src_trace_conf is relative to conf_dir by default + conf_dir = Path(self.env.app.confdir) + # if config toml file is used, src dir is relative to the config toml + if src_trace_sphinx_config.config_from_toml: + src_trace_toml_path = Path(src_trace_sphinx_config.config_from_toml) + conf_dir = conf_dir / src_trace_toml_path.parent + + src_dir = (conf_dir / src_trace_conf["src_dir"]).resolve() + return src_dir + + def render_needs( + self, + virtual_docs: VirtualDocs, + local_url_field: str | None, + remote_url_field: str | None, + dirs: dict[str, Path], + ) -> list[nodes.Node]: + """Render the needs from the virtual docs""" + rendered_needs: list[nodes.Node] = [] + for virtual_doc in virtual_docs.virtual_docs: + # # add source files into the dependency + # # https://www.sphinx-doc.org/en/master/extdev/envapi.html#sphinx.environment.BuildEnvironment.note_dependency + # self.env.note_dependency(str(virtual_doc.filepath.resolve())) + + filepath = virtual_doc.filepath + target_filepath = dirs["target_dir"] / filepath.relative_to(dirs["src_dir"]) + # mapping between lineno and need link in docs for local url + lineno_href = {} + # The link to the documentation page for the source file + docs_href = f"{dirs['out_dir'] / self.env.docname}.html" + if local_url_field: + # copy files to _build/html + target_filepath.parent.mkdir(parents=True, exist_ok=True) + target_filepath.write_text(filepath.read_text()) + for comment in virtual_doc.comments: + # Always generate link_name to avoid unbound errors + link_name = None + if local_url_field or remote_url_field: + # generate link name + link_name = generate_str_link_name( + comment, target_filepath, str(dirs["target_dir"]) + ) + if comment.resolved_marker: + # render needs from one-line marker + kwargs: dict[str, str | list[str]] = { + field_name: field_value + for field_name, field_value in comment.resolved_marker.items() + if field_name + not in [ + "title", + "type", + ] # title and type are mandatory for add_need() + } + + if local_url_field and link_name is not None: + kwargs[local_url_field] = link_name + if remote_url_field and link_name is not None: + kwargs[remote_url_field] = link_name + + marker_needs: list[nodes.Node] = add_need( + app=self.env.app, # The Sphinx application object + state=self.state, # The docutils state object + docname=self.env.docname, # The current document name + lineno=self.lineno, # The line number where the directive is used + need_type=str( + comment.resolved_marker["type"] + ), # The type of the need + title=str( + comment.resolved_marker["title"] + ), # The title of the need + **cast(dict[str, Any], kwargs), # type: ignore[misc] + ) + rendered_needs.extend(marker_needs) + if local_url_field: + # save the mapping of need links and line numbers of source codes + # for the later use in `html-collect-pages` + lineno_href[comment.start_line] = ( + f"{docs_href}#{comment.resolved_marker['id']}" + ) + + if local_url_field: + # save the mappings of need links and line numbers of source codes + # for the later use in `html-collect-pages` + file_lineno_href.mappings[str(target_filepath)] = lineno_href + + return rendered_needs diff --git a/src/sphinx_codelinks/sphinx_extension/html_wrapper.py b/src/sphinx_codelinks/sphinx_extension/html_wrapper.py new file mode 100644 index 0000000..2be89fc --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/html_wrapper.py @@ -0,0 +1,52 @@ +from collections.abc import Generator +from pathlib import Path +from typing import Any + +from pygments import highlight +from pygments.formatters import HtmlFormatter +from pygments.lexers import CLexer + + +class LineFormatter(HtmlFormatter): # type: ignore[type-arg] + def __init__(self, lineno_href: dict[int, str], *args: Any, **kwargs: Any) -> None: # type: ignore[misc] + super().__init__(*args, **kwargs) + self.lineno_href = lineno_href + + def wrap(self, source: Generator[Any]) -> Generator[Any]: # type: ignore[misc] + return self._wrap_custom_lines(super().wrap(source)) # type: ignore[no-untyped-call] + + def _wrap_custom_lines(self, source: Generator[Any]) -> Generator[Any]: # type: ignore[misc] + lineno = 0 + for is_line, line_html in source: + if is_line: + lineno += 1 + if lineno in self.lineno_href: + lineno_achor, inline_lineno, code_span = line_html.split("") + # Ensure the anchor is closed + inline_lineno = inline_lineno + "" + lineno_achor = lineno_achor + "" + # make the code as a link to the documentation + yield ( + is_line, + f'[docs]{line_html}', + ) + else: + yield is_line, f"{line_html}" + else: + yield is_line, line_html + + +def html_wrapper(filepath: Path, lineno_href: dict[int, str]) -> str: + code = filepath.read_text() + + formatter = LineFormatter( + lineno_href=lineno_href, + # use inline, as table may make lineno and code misaligned with certain Sphinx themes + linenos="inline", + lineanchors="L", # Adds anchor IDs like id="L-20" + anchorlinenos=True, # Makes line numbers clickable (link to #L-20) + wrapcode=False, # Wraps the code in a
with class "highlight" + ) + + html_content: str = highlight(code, CLexer(stripnl=False), formatter) + return html_content diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py new file mode 100644 index 0000000..b0df635 --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -0,0 +1,197 @@ +from collections.abc import Iterator # only in python 3.11 afterwards +import contextlib +from pathlib import Path +from timeit import default_timer as timer # Used for timing measurements +import tomllib +from typing import Any + +from sphinx.application import Sphinx +from sphinx.config import Config +from sphinx.environment import BuildEnvironment +from sphinx.util import logging +from sphinx.util.fileutil import copy_asset +from sphinx_needs.api import ( # type: ignore[import-untyped] + add_extra_option, + add_need_type, +) +from ubt_source_tracing.sphinx_extension import debug +from ubt_source_tracing.sphinx_extension.config import ( + SRC_TRACE_CACHE, + SrcTraceSphinxConfig, + file_lineno_href, +) +from ubt_source_tracing.sphinx_extension.directives.src_trace import ( + SourceTracing, + SourceTracingDirective, +) +from ubt_source_tracing.sphinx_extension.html_wrapper import html_wrapper +from ubt_source_tracing.virtual_docs.config import ( + OneLineCommentStyle, + OneLineCommentStyleType, +) +from ubt_source_tracing.virtual_docs.virtual_docs import VirtualDocs + +logger = logging.getLogger(__name__) + + +def setup(app: Sphinx) -> dict[str, Any]: # type: ignore[misc] + app.add_node(SourceTracing) + app.add_directive("src-trace", SourceTracingDirective) + SrcTraceSphinxConfig.add_config_values(app) + + app.connect("config-inited", load_config_from_toml, priority=10) + app.connect( + "config-inited", update_sn_extra_options, priority=11 + ) # run early otherwise, extra options are not set for nested_parse + app.connect("config-inited", update_sn_types) + + app.connect("env-before-read-docs", prepare_env) + app.connect("html-collect-pages", generate_code_page) + app.connect("html-page-context", add_custom_css) + app.connect("builder-inited", builder_inited) + app.connect("build-finished", emit_warnings) + app.connect("build-finished", debug.process_timing) + return { + "version": "builtin", + "parallel_read_safe": True, + "parallel_write_safe": True, + } + + +def builder_inited(app: Sphinx) -> None: + custom_css = Path(__file__).parent / "ub_sct.css" + copy_asset(custom_css, Path(app.outdir) / "_static" / "source_tracing") + + +def add_custom_css( # type: ignore[misc] + app: Sphinx, + pagename: str, + templatename: str, + context: dict[str, Any], + _doctree: Any, +) -> None: + target_htmls = { + str(Path(file_path).relative_to(app.outdir).with_suffix("")) + for file_path in file_lineno_href.mappings + } + + if pagename in target_htmls and templatename == "page.html": + if "css_files" not in context: + context["css_files"] = [] + context["css_files"].append( + "_static/source_tracing/ub_sct.css" + ) # Add the custom CSS file to the context + + +def generate_code_page( + app: Sphinx, +) -> Iterator[tuple[str, dict[str, str], str]] | None: + for file, lineno_href in file_lineno_href.mappings.items(): + file_path = Path(file) + pagename = str((file_path.relative_to(app.outdir)).with_suffix("")) + + html_content = html_wrapper( + file_path, + lineno_href=lineno_href, + ) + + context = { + "title": f"Source Code Tracing: {file_path.name}", + "body": html_content, + } + + yield pagename, context, "page.html" + + file_lineno_href.mappings.clear() # Clear the mappings after generating the pages + return None + + +def load_config_from_toml(app: Sphinx, config: Config) -> None: + """Load the configuration from a TOML file, if defined in conf.py.""" + src_trc_sphinx_config = SrcTraceSphinxConfig(config) + if src_trc_sphinx_config.config_from_toml is None: + return + + # resolve relative to confdir + toml_file = Path(app.confdir, src_trc_sphinx_config.config_from_toml).resolve() + # toml_path = src_trc_sphinx_config.from_toml_table + + if not toml_file.exists(): + logger.warning( + f"Source tracing configuration file {toml_file} does not exist. " + "Using configuration from conf.py." + ) + return + try: + with toml_file.open("rb") as f: + toml_data = tomllib.load(f) + toml_data = toml_data["src_trace"] + if not isinstance(toml_data, dict): + raise Exception(f"data must be a dict in {toml_file}") + + except Exception as e: + logger.warning( + f"Failed to load source tracing configuration from {toml_file}: {e}" + ) + return + + allowed_keys = SrcTraceSphinxConfig.field_names() + for key, value in toml_data.items(): + if key not in allowed_keys: + continue + if key == "projects": + for project_config in value.values(): + oneline_comment_style: OneLineCommentStyleType | None = ( + project_config.get("oneline_comment_style") + ) + if oneline_comment_style: + project_config["oneline_comment_style"] = OneLineCommentStyle( + **project_config["oneline_comment_style"] + ) + config[f"src_trace_{key}"] = value + + +def update_sn_extra_options(app: Sphinx, config: Config) -> None: + src_trace_sphinx_config = SrcTraceSphinxConfig(config) + add_extra_option(app, "project") + add_extra_option(app, "file") + add_extra_option(app, "directory") + if src_trace_sphinx_config.set_local_url: + add_extra_option(app, src_trace_sphinx_config.local_url_field) + if src_trace_sphinx_config.set_remote_url: + add_extra_option(app, src_trace_sphinx_config.remote_url_field) + + +def update_sn_types(app: Sphinx, _config: Config) -> None: + add_need_type(app, "srctrace", "Src-Trace", "ST_", "#ffffff", "node") + + +def prepare_env(app: Sphinx, env: BuildEnvironment, _docnames: list[str]) -> None: # noqa: ARG001 # required by Sphinx + """ + Prepares the sphinx environment to store stc-trace internal data. + """ + src_trace_sphinx_config = SrcTraceSphinxConfig(app.config) + + # Set time measurement flag + if src_trace_sphinx_config.debug_measurement: + debug.START_TIME = timer() # Store the rough start time of Sphinx build + debug.EXECUTE_TIME_MEASUREMENTS = True + + if src_trace_sphinx_config.debug_filters: + with contextlib.suppress(FileNotFoundError): + Path(str(app.outdir), "debug_filters.jsonl").unlink() + + +def emit_warnings( + app: Sphinx, + _env: BuildEnvironment, +) -> None: + warnings = VirtualDocs.load_warnings(Path(app.outdir) / SRC_TRACE_CACHE) + if not warnings: + return + for warning in warnings: + logger.warning( + f"{warning.file_path}:{warning.lineno}: {warning.msg}", + type=warning.type, + subtype=warning.sub_type, + ) diff --git a/src/sphinx_codelinks/sphinx_extension/ub_sct.css b/src/sphinx_codelinks/sphinx_extension/ub_sct.css new file mode 100644 index 0000000..8e6941d --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/ub_sct.css @@ -0,0 +1,8 @@ +.highlight { + position:relative +} + +.viewcode-back{ + position: absolute; + right:0; +} diff --git a/src/sphinx_codelinks/virtual_docs/config.py b/src/sphinx_codelinks/virtual_docs/config.py new file mode 100644 index 0000000..dd8bd6b --- /dev/null +++ b/src/sphinx_codelinks/virtual_docs/config.py @@ -0,0 +1,204 @@ +from dataclasses import MISSING, dataclass, field, fields +import logging +import os +from typing import Any, Literal, TypedDict, cast + +from jsonschema import ValidationError, validate + +# initialize logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# log to the console +console = logging.StreamHandler() +console.setLevel(logging.INFO) +logger.addHandler(console) + +ESCAPE = "\\" +SUPPORTED_COMMENT_TYPES = {"c", "h", "cpp", "hpp"} + + +class FieldConfig(TypedDict, total=False): + name: str + quoted: bool + named: bool + type: Literal["str", "list[str]"] + default: str | list[str] | None + + +class OneLineCommentStyleType(TypedDict): + start_sequence: str + end_sequence: str + field_split_char: str + needs_fields: list[FieldConfig] + + +@dataclass +class OneLineCommentStyle: + def __setattr__(self, name: str, value: Any) -> None: # type: ignore[misc] + if name == "needs_fields": + # apply default to fields + self.apply_needs_field_default(value) + return super().__setattr__(name, value) + + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + start_sequence: str = field(default="@", metadata={"schema": {"type": "string"}}) + """Chars sequence to indicate the start of the one-line comment.""" + + end_sequence: str = field( + default=os.linesep, metadata={"schema": {"type": "string"}} + ) + """Chars sequence to indicate the end of the one-line comment.""" + + field_split_char: str = field(default=",", metadata={"schema": {"type": "string"}}) + """Char sequence to split the fields.""" + + needs_fields: list[FieldConfig] = field( + default_factory=lambda: [ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + metadata={ + "required_fields": ["title", "type"], + "field_default": { + "type": "str", + "quoted": False, + "named": False, + }, + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "quoted": {"type": "boolean", "default": False}, + "named": {"type": "boolean", "default": False}, + "type": { + "type": "string", + "enum": ["str", "list[str]"], + "default": "str", + }, + "default": { + "anyOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}}, + ] + }, + }, + "required": ["name"], + "additionalProperties": False, + "allOf": [ + { + "if": {"properties": {"type": {"const": "list[str]"}}}, + "then": { + "properties": { + "default": { + "type": "array", + "items": {"type": "string"}, + } + } + }, + }, + { + "if": {"properties": {"type": {"const": "str"}}}, + "then": {"properties": {"default": {"type": "string"}}}, + }, + ], + }, + }, + }, + ) + + @classmethod + def apply_needs_field_default(cls, given_fields: list[FieldConfig]) -> None: + field_default = next( + _field.metadata["field_default"] + for _field in fields(cls) + if _field.name == "needs_fields" + ) + + for _field in given_fields: + for _default in field_default: + if _default not in _field: + _field[_default] = field_default[_default] # type: ignore[literal-required] # dynamically assign keys + + @classmethod + def get_required_fields(cls, name: str) -> list[str] | None: + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING: + return cast(list[str], _field.metadata["required_fields"]) + return None + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[misc] + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[misc] + return None + + def check_schema(self) -> list[str]: + errors = [] + for _field_name in self.field_names(): + schema = self.get_schema(_field_name) + value = getattr(self, _field_name) + try: + validate(instance=value, schema=schema) # type: ignore[arg-type] # validate has no type specified + except ValidationError as e: + if _field_name == "needs_fields": + need_field_name = value[e.path[0]]["name"] + errors.append( + f"Schema validation error in need_fields '{need_field_name}': {e.message}" + ) + else: + errors.append( + f"Schema validation error in field '{_field_name}': {e.message}" + ) + return errors + + def check_required_fields(self) -> list[str]: + errors = [] + required_fields = self.get_required_fields("needs_fields") + if required_fields is None: + errors.append("No required fields specified.") + return errors + for _field in self.needs_fields: + if _field["name"] in required_fields: + required_fields.remove(_field["name"]) + if len(required_fields) != 0: + errors.append(f"Missing required fields: {required_fields}") + + return errors + + def check_fields_mutually_exclusive(self) -> list[str]: + errors = [] + needs_field_names = set() + for _field in self.needs_fields: + if _field["name"] in needs_field_names: + errors.append(f"Field '{_field['name']}' is defined multiple times.") + needs_field_names.add(_field["name"]) + return errors + + def check_fields_configuration(self) -> list[str]: + return ( + self.check_schema() + + self.check_required_fields() + + self.check_fields_mutually_exclusive() + ) + + def get_cnt_required_fields(self) -> int: + cnt_required_fields = 0 + for _field in self.needs_fields: + if _field.get("default") is None: + cnt_required_fields += 1 + return cnt_required_fields + + def get_pos_list_str(self) -> list[int]: + pos_list_str = [] + for idx, _field in enumerate(self.needs_fields): + if _field["type"] == "list[str]": + pos_list_str.append(idx + 1) + return pos_list_str diff --git a/src/sphinx_codelinks/virtual_docs/ubt_models.py b/src/sphinx_codelinks/virtual_docs/ubt_models.py new file mode 100644 index 0000000..d18cf3e --- /dev/null +++ b/src/sphinx_codelinks/virtual_docs/ubt_models.py @@ -0,0 +1,125 @@ +import json +from pathlib import Path +from typing import cast + + +class MultipleMarkerError(Exception): + """Custom exception for multiple markers in a comment.""" + + +class UBTComment: + """Wrap Comment object from comment_parser.""" + + def __init__( + self, + text: str, + start_line: int, + resolved_marker: dict[str, str | list[str]], + marker_type: str = "oneline", + ): + self.text = text + # start and end columns are not supported by comment_parser + # start_line and end_line are the line number of the comment by default. + # If the marked text exists, they will be line numbers of that. + self.start_line = start_line + # so far only one-line comment is taken. end_line is kept for the future multi-line styles + self.end_line = ( + self.start_line + self.text.count("\n") - 1 + if self.text.count("\n") + else self.start_line + ) + self.resolved_marker: dict[str, str | list[str]] = resolved_marker + self.marker_type: str = marker_type + + def __eq__(self, value): + if isinstance(value, UBTComment): + return self.__dict__ == value.__dict__ + return False + + def to_dict(self) -> dict[str, dict[str, str | list[str]] | str | int]: + return { + "text": self.text, + "start_line": self.start_line, + "end_line": self.end_line, + "marker_type": self.marker_type, + "resolved_marker": self.resolved_marker, + } + + +class UBTSourceFile: + def __init__( + self, + filepath: Path, + src_dir: Path, + comments: list[UBTComment] | None = None, + output_dir: str = "./", + ): + self.filepath: Path = filepath + self.src_dir: Path = src_dir + self.comments: list[UBTComment] = [] + if comments: + self.comments.extend(comments) + self.output_dir = Path(output_dir) + self.changed_date = self.filepath.stat().st_mtime + + def __eq__(self, value): + if isinstance(value, UBTSourceFile): + return self.__dict__ == value.__dict__ + return False + + def add_comment(self, comment: UBTComment) -> None: + self.comments.append(comment) + + def add_comments(self, comment: list[UBTComment]) -> None: + self.comments.extend(comment) + + def to_json(self) -> None: + comments = [comment.__dict__ for comment in self.comments] + output_path = self.output_dir / self.filepath.with_suffix(".json").relative_to( + self.src_dir + ) + if not output_path.parent.exists(): + output_path.parent.mkdir(parents=True) + with output_path.open("w") as f: + json.dump(comments, f) + + +class UBTCache: + def __init__( + self, + cache_path: str = "./ubt_cache.json", + uncached_files: list[UBTSourceFile] | None = None, + ): + if uncached_files is None: + uncached_files = [] + self.cache_path = Path(cache_path) + self.uncached_files = uncached_files + self.cached_files = self.load_cache() + + def load_cache(self) -> dict[str, float]: + if not self.cache_path.exists(): + return {} + with self.cache_path.open("r") as f: + cached_files = cast(dict[str, float], json.load(f)) + return cached_files + + def add_uncached_files(self, uncached_files: list[UBTSourceFile]) -> None: + self.uncached_files.extend(uncached_files) + for uncached_file in uncached_files: + if str(uncached_file.filepath) in self.cached_files: + self.cached_files.pop(str(uncached_file.filepath)) + + def update_cache(self) -> None: + for src_file in self.uncached_files: + if ( + str(src_file.filepath) in self.cached_files + and src_file.changed_date == self.cached_files[str(src_file.filepath)] + ): + continue + self.cached_files[str(src_file.filepath)] = src_file.changed_date + # remove cached files from uncached_files + self.uncached_files = [] + if not self.cache_path.parent.exists(): + self.cache_path.parent.mkdir(parents=True) + with self.cache_path.open("w") as f: + json.dump(self.cached_files, f) diff --git a/src/sphinx_codelinks/virtual_docs/utils.py b/src/sphinx_codelinks/virtual_docs/utils.py new file mode 100644 index 0000000..3fb8425 --- /dev/null +++ b/src/sphinx_codelinks/virtual_docs/utils.py @@ -0,0 +1,215 @@ +from dataclasses import dataclass +from enum import Enum +import logging +import os + +from ubt_source_tracing.virtual_docs.config import ( + ESCAPE, + SUPPORTED_COMMENT_TYPES, + OneLineCommentStyle, +) + +# initialize logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# log to the console +console = logging.StreamHandler() +console.setLevel(logging.INFO) +logger.addHandler(console) + + +class WarningSubTypeEnum(str, Enum): + """Enum for warning sub types.""" + + too_many_fields = "too_many_fields" + too_few_fields = "too_few_fields" + missing_square_brackets = "missing_square_brackets" + not_start_or_end_with_square_brackets = "not_start_or_end_with_square_brackets" + newline_in_field = "newline_in_field" + + +@dataclass +class OnelineParserInvalidWarning: + """Invalid oneline comments.""" + + sub_type: WarningSubTypeEnum + msg: str + + +def oneline_parser( # noqa: PLR0912, PLR0911 # handel warnings + oneline: str, oneline_config: OneLineCommentStyle +) -> dict[str, str | list[str]] | OnelineParserInvalidWarning | None: + """ + Extract the string from the custom one-line comment style with the following steps. + + - Locate the start and end sequences + - extract the string between them + - apply custom_split to split the strings into a list of fields by `field_split_char` + - check the number of required fields and the max number of the given fields + - split the strings located in the field with `type: list[str]` to a list of string + - introduce the default values to those fields which are not given + """ + # find indices start and end char + start_idx = oneline.find(oneline_config.start_sequence) + end_idx = oneline.rfind(oneline_config.end_sequence) + if start_idx == -1 or end_idx == -1: + # start or end sequences do not exist + return None + + # extract the string wrapped by start and end + string = oneline[start_idx + len(oneline_config.start_sequence) : end_idx] + + # numbers of needs_fields which are required + cnt_required_fields = oneline_config.get_cnt_required_fields() + # indices of the field which has type:list[str] + positions_list_str = oneline_config.get_pos_list_str() + + min_fields = cnt_required_fields + max_fields = len(oneline_config.needs_fields) + + string_fields = [ + _field.strip(" ") + for _field in custom_split( + string, oneline_config.field_split_char, positions_list_str + ) + ] + if len(string_fields) < min_fields: + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.too_few_fields, + msg=f"{len(string_fields)} given fields. They shall be more than {min_fields}", + ) + + if len(string_fields) > max_fields: + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.too_many_fields, + msg=f"{len(string_fields)} given fields. They shall be less than {max_fields}", + ) + resolved: dict[str, str | list[str]] = {} + for idx in range(len(oneline_config.needs_fields)): + field_name: str = oneline_config.needs_fields[idx]["name"] + if len(string_fields) > idx: + # given fields + if is_newline_in_field(string_fields[idx]): + # the case where the field contains a new line character + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.newline_in_field, + msg=f"Field {field_name} has newline character. It is not allowed", + ) + if oneline_config.needs_fields[idx]["type"] == "str": + resolved[field_name] = string_fields[idx] + elif oneline_config.needs_fields[idx]["type"] == "list[str]": + # find the indices of "[" and "]" + start_idx = string_fields[idx].find("[") + end_idx = string_fields[idx].rfind("]") + if start_idx == -1 or end_idx == -1: + # brackets are not found + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.missing_square_brackets, + msg=f"Field {field_name} with 'type': '{oneline_config.needs_fields[idx]['type']}' must be given with '[]' brackets", + ) + + if start_idx != 0 or end_idx != len(string_fields[idx]) - 1: + # brackets are found but not at the beginning and the end + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.not_start_or_end_with_square_brackets, + msg=f"Field {field_name} with 'type': '{oneline_config.needs_fields[idx]['type']}' must start with '[' and end with ']'", + ) + + string_items = string_fields[idx][start_idx + 1 : end_idx] + + if not string_items.strip(): + # the case where the empty string ("") or only spaces between "[" "]" + resolved[field_name] = [] + else: + items = [_item.strip() for _item in custom_split(string_items, ",")] + resolved[field_name] = [item.strip() for item in items] + else: + # for not given fields, introduce the default + default = oneline_config.needs_fields[idx].get("default") + if default is None: + continue + resolved[field_name] = default + + return resolved + + +def custom_split( + string: str, delimiter: str, positions_list_str: list[int] | None = None +) -> list[str]: + """ + A string shall be split with the following conditions: + + - To use special chars in literal , escape ('\') must be used + - String shall be split by the given delimiter + - In a field with `type: str`: + - Special chars are delimiter, '\', '[' and ']' + - In a field with `type: list[str]`: + - Special chars are only '[' and ']' + + When the string is given without any fields with `type: list[str]` (positions_list_str=None), + it's considered as it is in a field with `type: str`. + """ + if positions_list_str is None: + positions_list_str = [] + escape_chars = [delimiter, "[", "]", ESCAPE] + field = [] # a list of string for a field + fields: list[str] = [] # a list of string which contains + leading_escape = False + expect_closing_bracket = False + + for char in string: + # +1 to locate the current field position + current_field_idx = len(fields) + 1 + is_list_str_field = current_field_idx in positions_list_str + + if leading_escape: + if char not in escape_chars: + # leading escape is considered as a literal + field.append(ESCAPE) + field.append(char) + leading_escape = False + continue + + if char == ESCAPE and not is_list_str_field: + leading_escape = True + continue + + if char == delimiter: + if is_list_str_field and expect_closing_bracket: + # delimiter occurs in the field with type:list[str] + field.append(char) + else: + fields.append("".join(field)) + field = [] + continue + + if is_list_str_field: + if char == "[": + expect_closing_bracket = True + if char == "]": + expect_closing_bracket = False + + field.append(char) + + # add last field + fields.append("".join(field)) + return fields + + +def is_newline_in_field(field: str) -> bool: + """ + Check if the field contains a new line character. + """ + return os.linesep in field + + +def get_file_types(comment_type: str) -> list[str] | None: + """ + Get the list of file types to be discovered. + """ + file_types = ( + list(SUPPORTED_COMMENT_TYPES) + if comment_type in SUPPORTED_COMMENT_TYPES + else None + ) + return file_types diff --git a/src/sphinx_codelinks/virtual_docs/virtual_docs.py b/src/sphinx_codelinks/virtual_docs/virtual_docs.py new file mode 100644 index 0000000..6aec064 --- /dev/null +++ b/src/sphinx_codelinks/virtual_docs/virtual_docs.py @@ -0,0 +1,241 @@ +from dataclasses import dataclass +import json +import logging +import os +from pathlib import Path + +from comment_parser.parsers.common import Comment +from ubt_source_tracing.virtual_docs.config import ( + SUPPORTED_COMMENT_TYPES, + OneLineCommentStyle, +) +from ubt_source_tracing.virtual_docs.ubt_models import ( + UBTCache, + UBTComment, + UBTSourceFile, +) +from ubt_source_tracing.virtual_docs.utils import ( + OnelineParserInvalidWarning, + oneline_parser, +) + +# initialize logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# log to the console +console = logging.StreamHandler() +console.setLevel(logging.INFO) +logger.addHandler(console) + + +@dataclass +class VirtualDocsOneLineWarning: + file_path: str + lineno: int + msg: str + type: str + sub_type: str + + +class VirtualDocs: + warning_filepath: Path = Path("cached_warnings") / "vdocs_warnings.json" + + def __init__( + self, + src_files: list[Path], + src_dir: str, + output_dir: str, + oneline_comment_style: OneLineCommentStyle, + comment_type: str = "c", + ) -> None: + self.src_files = src_files + self.src_dir = Path(src_dir) + self.output_dir = Path(output_dir) + self.comment_type = comment_type + self.cache = UBTCache(str(self.output_dir / "ubt_cache.json")) + self.cache.add_uncached_files(self._uncached_files()) + self.virtual_docs: list[UBTSourceFile] = [] + self.oneline_comment_style = oneline_comment_style + self.oneline_warnings: list[VirtualDocsOneLineWarning] = [] + self.warnings_path = self.output_dir / VirtualDocs.warning_filepath + + def collect(self) -> None: + # import C parser to avoid `python-magic` dependency + # https://github.com/jeanralphaviles/comment_parser?tab=readme-ov-file#osx-and-windows + if self.comment_type not in SUPPORTED_COMMENT_TYPES: + raise Exception( + f"Unsupported comment type: {self.comment_type}. Supported types are: {SUPPORTED_COMMENT_TYPES}." + ) + from comment_parser.parsers.c_parser import ( # type: ignore[import-untyped] + extract_comments, + ) + + virtual_docs = [] + self.load_virtual_docs() + # parse all uncached files + for src_file in self.cache.uncached_files: + ml_comments: list[Comment] = [] + oneline_comments: list[Comment] = [] + with src_file.filepath.open("r", encoding="utf-8") as code: + comments = extract_comments(code.read()) + + # separate one-line and multi-line comments + for comment in comments: + if comment.is_multiline(): + ml_comments.append(comment) + else: + oneline_comments.append(comment) + + # break all multi-line comments to single-line comments + for comment in ml_comments: + single_lines = comment.text().splitlines() + for idx, line in enumerate(single_lines): + oneline_comments.append(Comment(line, comment.line_number() + idx)) # type: ignore[call-arg] + + ubt_comments: list[UBTComment] = [] + + for comment in oneline_comments: + resolved = oneline_parser( + f"{comment.text()}{os.linesep}", self.oneline_comment_style + ) + + if isinstance(resolved, OnelineParserInvalidWarning): + self.oneline_warnings.append( + VirtualDocsOneLineWarning( + str(src_file.filepath), + comment.line_number(), + resolved.msg, + type="oneline", + sub_type=resolved.sub_type.value, + ) + ) + continue + + if resolved: + ubt_comments.append( + UBTComment( + f"{comment.text()}{os.linesep}", + start_line=comment.line_number(), + resolved_marker=resolved, + ) + ) + + ubt_comments.sort(key=lambda x: x.start_line) + src_file.add_comments(ubt_comments) + virtual_docs.append(src_file) + + self.virtual_docs.extend(virtual_docs) + self.update_warnings() + + def _uncached_files(self) -> list[UBTSourceFile]: + uncached_files = [] + for src_file in self.src_files: + ubt_src_file = UBTSourceFile( + src_file, self.src_dir, output_dir=str(self.output_dir) + ) + # check cached virtual documents + if ( + str(src_file) in self.cache.cached_files + and ( + self.output_dir + / (src_file.with_suffix(".json").relative_to(self.src_dir)) + ).exists() + and self.cache.cached_files.get(str(ubt_src_file.filepath)) + == ubt_src_file.changed_date + ): + continue + uncached_files.append(ubt_src_file) + return uncached_files + + def dump_virtual_docs(self) -> None: + for src_file in self.virtual_docs: + src_file.to_json() + + @classmethod + def load_warnings( + cls, warnings_dir: Path + ) -> list[VirtualDocsOneLineWarning] | None: + """Load warnings from the given path. + + It mainly used for other apps or users to load warnings files directly. + """ + warnings_path = warnings_dir / cls.warning_filepath + if not warnings_path.exists(): + return None + with warnings_path.open("r") as f: + # load the json file and convert to VirtualDocsOneLineWarning] + warnings = json.load(f) + loaded_warnings = [ + VirtualDocsOneLineWarning(**warning) for warning in warnings + ] + return loaded_warnings + + def _load_warnings(self) -> list[VirtualDocsOneLineWarning] | None: + if not self.warnings_path.exists(): + return None + with self.warnings_path.open("r") as f: + # load the json file and convert to VirtualDocsOneLineWarning] + warnings = json.load(f) + loaded_warnings = [ + VirtualDocsOneLineWarning(**warning) for warning in warnings + ] + return loaded_warnings + + def update_warnings(self) -> None: + loaded_warnings = self._load_warnings() + current_warnings = [_warning.__dict__ for _warning in self.oneline_warnings] + if loaded_warnings: + _warnings = [_warning.__dict__ for _warning in loaded_warnings] + cached_warnings = [ + _warning + for _warning in _warnings + if not ( + _warning["file_path"] + in [str(src_file) for src_file in self.src_files] + and _warning["file_path"] + in [ + str(ubt_src_file.filepath) + for ubt_src_file in self.cache.uncached_files + ] + ) + ] + total_warning = cached_warnings + current_warnings + else: + total_warning = current_warnings + if not self.warnings_path.parent.exists(): + self.warnings_path.parent.mkdir(parents=True) + with self.warnings_path.open("w") as f: + json.dump( + total_warning, + f, + ) + + def load_virtual_docs(self) -> None: + # only load cached files that are in the self.src_files + src_files = [ + src_file + for src_file in self.cache.cached_files + if src_file in [str(file_path) for file_path in self.src_files] + ] + for src_file in src_files: + src_path = Path(src_file) + virt_doc_path = self.output_dir / ( + src_path.with_suffix(".json").relative_to(self.src_dir) + ) + if virt_doc_path.exists(): + ubt_src_file = UBTSourceFile( + src_path, self.src_dir, output_dir=str(self.output_dir) + ) + comments = json.load(virt_doc_path.open("r")) + ubt_src_file.add_comments( + [ + UBTComment( + text=comment["text"], + start_line=comment["start_line"], + resolved_marker=comment["resolved_marker"], + marker_type=comment["marker_type"], + ) + for comment in comments + ] + ) + self.virtual_docs.append(ubt_src_file) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project0-source_code0].doctree.xml b/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project0-source_code0].doctree.xml new file mode 100644 index 0000000..edf41b7 --- /dev/null +++ b/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project0-source_code0].doctree.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project1-source_code1].doctree.xml b/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project1-source_code1].doctree.xml new file mode 100644 index 0000000..ce9b94a --- /dev/null +++ b/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project1-source_code1].doctree.xml @@ -0,0 +1,3 @@ + + + diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..09be6d6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,63 @@ +from pathlib import Path + +from docutils.nodes import document +import pytest +from syrupy.extensions.single_file import SingleFileSnapshotExtension, WriteMode + +TEST_DIR = Path(__file__).parent +SRC_TRACE_TOML = TEST_DIR / "data" / "sphinx" / "src_trace.toml" +BASIC_VDOC_TOML = TEST_DIR / "data" / "oneline_comment_basic" / "vdoc_config.toml" +DEFAULT_VDOC_TOML = TEST_DIR / "data" / "oneline_comment_default" / "vdoc_config.toml" +RECURSIVE_DIR_VDOC_TOML = TEST_DIR / "doc_test" / "recursive_dirs" / "src_trace.toml" + + +@pytest.fixture(scope="session") +def source_directory() -> Path: + tests_dir = Path(__file__).parent + source_directory = tests_dir / "data" / "dcdc" + return source_directory + + +@pytest.fixture(scope="session") +def source_paths(source_directory: Path) -> list[Path]: + source_paths = [ + source_directory / "charge" / "demo_1.cpp", + source_directory / "charge" / "demo_2.cpp", + source_directory / "discharge" / "demo_3.cpp", + source_directory / "supercharge.cpp", + ] + return source_paths + + +@pytest.fixture(scope="session", autouse=True) +def temporary_gitignore(source_directory: Path): + gitignore_path = source_directory / ".gitignore" + gitignore_path.write_text("demo_1.cpp\n", encoding="utf-8") + yield + gitignore_path.unlink() + + +class DoctreeSnapshotExtension(SingleFileSnapshotExtension): + _write_mode = WriteMode.TEXT + _file_extension = "doctree.xml" + + def serialize(self, data, **_kwargs): + if not isinstance(data, document): + raise TypeError(f"Expected document, got {type(data)}") + doc = data.deepcopy() + doc["source"] = "" # this will be a temp path + doc.attributes.pop("translation_progress", None) # added in sphinx 7.1 + return doc.pformat() + + +@pytest.fixture +def snapshot_doctree(snapshot): + """Snapshot fixture for doctrees. + + Here we try to sanitize the doctree, to make the snapshots reproducible. + """ + try: + return snapshot.with_defaults(extension_class=DoctreeSnapshotExtension) + except AttributeError: + # fallback for older versions of pytest-snapshot + return snapshot.use_extension(DoctreeSnapshotExtension) diff --git a/tests/data/dcdc/charge/demo_1.cpp b/tests/data/dcdc/charge/demo_1.cpp new file mode 100644 index 0000000..750ffdd --- /dev/null +++ b/tests/data/dcdc/charge/demo_1.cpp @@ -0,0 +1,28 @@ +// demo_1.cpp + +/** + * @file another_example.cpp + * @brief Test file with nested rst blocks. + */ + + #include + + /** + * @brief Function with nested reST blocks. + * + * Include details on how to handle edge cases. + * + * Additional processing steps here. + * [[IMPL_processAssemble, processAssemble function]] + */ + void processAssemble(){ + //... + } + + // [[IMPL_main_demo1, main function]] + int main() { + std::cout << "Starting demo_1..." << std::endl; + processAssemble(); + std::cout << "Demo_1 finished." << std::endl; + return 0; + } diff --git a/tests/data/dcdc/charge/demo_2.cpp b/tests/data/dcdc/charge/demo_2.cpp new file mode 100644 index 0000000..8904a1a --- /dev/null +++ b/tests/data/dcdc/charge/demo_2.cpp @@ -0,0 +1,48 @@ +// demo_2.cpp + +/** + * @file another_example.cpp + * @brief Test file with nested rst blocks. + */ + + #include + + /** + * @brief Function with nested reST blocks. + * + * + * Include details on how to handle edge cases. + * + * Additional processing steps here. + * [[IMPL_filterData, filterData func, impl]] + */ + void filterData() { + // ... implementation ... + } + + /** + * @brief Function with multiple rst blocks. + * + * Some code here. + * + * Feature F - Data visualization + * [[IMPL_processAggregate]] + */ + void processAggregate(){ + //... + } + + /** + * @brief Function with a rst blocks. + * .. impl:: Feature G - Data loss prevention + * + * Some description here. + * [[ IMPL_main_demo2, main func in demo_2]] + */ + int main() { + std::cout << "Starting demo_2..." << std::endl; + filterData(); + processAggregate(); + std::cout << "Demo_2 finished." << std::endl; + return 0; + } diff --git a/tests/data/dcdc/discharge/demo_3.cpp b/tests/data/dcdc/discharge/demo_3.cpp new file mode 100644 index 0000000..9a635fc --- /dev/null +++ b/tests/data/dcdc/discharge/demo_3.cpp @@ -0,0 +1,56 @@ +// demo_3.cpp + +/** + * @file varied_example.cpp + * @brief Test file with varied rst formatting. + */ + + #include + + // GLOBAL REQUIRE: Feature G - Configuration management + // Description: Manage application configuration. + + /** + * @brief Function with varied rst spacing. + * + * + */ + void logErrors() { + // ... implementation ... + } + + /** + * @brief Function with rst on same line. (NOT valid) + * + */ +// [[ IMPL_displayUI, displayUI() func\, so that it displays UI]] + void displayUI(){ + //... + } + + /** + * @brief function with rst with extra space. + * + * \rst + * .. impl:: Feature J - Data backup + * :id: IMPL_6 + * + * Backup user data. + * \endrst + * + * // TODO: Improve backup performance. + * // [[ IMPL_backupData, back up data]] + */ + void backupData(){ + //... + } + + // [[IMPL_main_demo_3, main func in demo_3.cpp]] + int main() { + std::cout << "Starting demo_3..." << std::endl; + logErrors(); + displayUI(); + backupData(); + std::cout << "Demo_3 finished." << std::endl; + return 0; + } diff --git a/tests/data/dcdc/supercharge.cpp b/tests/data/dcdc/supercharge.cpp new file mode 100644 index 0000000..24bc4e2 --- /dev/null +++ b/tests/data/dcdc/supercharge.cpp @@ -0,0 +1,31 @@ + +#include + +// [[IMPL_singleLineExample, singleLineExample func, impl, [IMPL_main_demo_3, IMPL_main_demo2]]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} + +/* + * + * This is a multi-line comment example. + * It spans multiple lines and contains detailed information. + * [[IMPL_multiLineExample, multiLineExample func, impl, [IMPL_main_demo_3, IMPL_main_demo1]]] + */ +void multiLineExample() +{ + std::cout << "Multi-line comment example" << std::endl; +} + +// [[IMPL_14, title 13, impl, 13[\[SPEC\,_1\]], open, low, high]] +// invalid because the too many fields +void baz() {} + +// one-line comment style: [[IMPL_main_supercharge, main func in supercharge.cpp, impl, [IMPL_main_demo_3, IMPL_main_demo1, IMPL_main_demo2]]] +int main() +{ + singleLineExample(); + multiLineExample(); + return 0; +} diff --git a/tests/data/oneline_comment_basic/basic_oneliners.c b/tests/data/oneline_comment_basic/basic_oneliners.c new file mode 100644 index 0000000..9102964 --- /dev/null +++ b/tests/data/oneline_comment_basic/basic_oneliners.c @@ -0,0 +1,29 @@ +// [[IMPL_1, Function Foo]] +void foo() {} + +// [[IMPL_2, Function Bar, impl, [], closed]] +void bar() {} + +// [[IMPL_3, Function Baz\, as I want it, impl, [], closed]] +void baz() {} + +// [[IMPL_5, Function Bar, impl, [SPEC_1, SPEC_2], open]] +void bar() {} + +// [[IMPL_6, Function Bar, impl, [SPEC_1, SPEC_2], [open]]] +// valid because "[open]" is parsed into field status +void foo() {} + +// [[IMPL_7, Function has a, in the title]] +// title has a non-escaped split char ',' +// type will be set to 'in the title' (will lead to a SN error) +void baz() {} + +// [[IMPL_8, [Title starts with a bracket], impl]] +// valid because title is of type string, it will be parsed to +// '[Title starts with a bracket]' +void baz() {} + +// [[IMPL_9, Function Baz, impl, [SPEC_1, SPEC_2[text], SPEC_3], open]] +// valid because the 2nd link item is 'SPEC_2]' +void baz() {} diff --git a/tests/data/oneline_comment_basic/vdoc_config.toml b/tests/data/oneline_comment_basic/vdoc_config.toml new file mode 100644 index 0000000..dec2a20 --- /dev/null +++ b/tests/data/oneline_comment_basic/vdoc_config.toml @@ -0,0 +1,19 @@ +src_dir = "./" +comment_type = "c" +exclude = [] +include = ["**/*.c", "**/*.h"] +gitignore = false + +[oneline_comment_style] +start_sequence = "[[" +end_sequence = "]]" # default is newline character +field_split_char = "," +needs_fields = [ + { "name" = "id" }, + { "name" = "title" }, + { "name" = "type", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, + { "name" = "status", "default" = "open" }, + { "name" = "priority", "default" = "low" }, +] diff --git a/tests/data/oneline_comment_default/default_oneliners.c b/tests/data/oneline_comment_default/default_oneliners.c new file mode 100644 index 0000000..5a7e920 --- /dev/null +++ b/tests/data/oneline_comment_default/default_oneliners.c @@ -0,0 +1,19 @@ +// @Function Foo, IMPL_1 +void foo() {} + +// @Function Bar, IMPL_2 +void bar() {} + +// @Function Baz\, as I want it, IMPL_3 +void baz() {} + +// @Function Bar\, , IMPL_4, impl, [SPEC_1, SPEC_2] +void bar() {} + +/* +* Multiple lines comment +* +* +* @Function Bar, , IMPL_4, impl, [SPEC_1, SPEC_2] +*/ +void bar() {} diff --git a/tests/data/oneline_comment_default/vdoc_config.toml b/tests/data/oneline_comment_default/vdoc_config.toml new file mode 100644 index 0000000..58183fb --- /dev/null +++ b/tests/data/oneline_comment_default/vdoc_config.toml @@ -0,0 +1,5 @@ +src_dir = "./" +comment_type = "c" +exclude = [] +include = ["**/*.c", "**/*.h"] +gitignore = false diff --git a/tests/data/sphinx/Makefile b/tests/data/sphinx/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/tests/data/sphinx/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/tests/data/sphinx/conf.py b/tests/data/sphinx/conf.py new file mode 100644 index 0000000..9918d4f --- /dev/null +++ b/tests/data/sphinx/conf.py @@ -0,0 +1,113 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "test_parse" +copyright = "2025, useblocks" +author = "team useblocks" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "furo" +# html_static_path = ["_static"] + +extensions = ["sphinx_needs", "ubt_source_tracing"] + +needs_types = [ + { + "directive": "story", + "title": "User Story", + "prefix": "US_", + "color": "#BFD8D2", + "style": "node", + }, + { + "directive": "spec", + "title": "Specification", + "prefix": "SP_", + "color": "#FEDCD2", + "style": "node", + }, + { + "directive": "implement", + "title": "Implementation", + "prefix": "IM_", + "color": "#DF744A", + "style": "node", + }, + { + "directive": "impl", + "title": "Impl", + "prefix": "IMPL_", + "color": "#DFd44A", + "style": "node", + }, + { + "directive": "test", + "title": "Test Case", + "prefix": "TC_", + "color": "#DCB239", + "style": "node", + }, +] + +needs_extra_options = ["priority"] + +src_trace_config_from_toml = "src_trace.toml" + +# # TODO implement me +# src_trace_set_local_url = True +# src_trace_local_url_field = "local-url" +# src_trace_set_remote_url = True +# src_trace_remote_url_field = "remote-url" + +# src_trace_projects = { +# # TODO use the key (add it to the src-trace need) +# "dcdc": { +# "type": "cpp", +# "src_dir": "../../dcdc", # relative to confdir +# "remote_url_pattern": "https://github.com/useblocks/ubtrace/blob/{commit}/{path}#L{line}", # optional +# "exclude": ["dcdc/src/ubt/ubt.cpp"], +# "include": ["**/*.cpp", "**/*.hpp"], # has default for each type +# "gitignore": True, # default is True +# # Proposal for the one-line comment style: +# # a need object defined in a one-line comment with the customized style. +# # The example is the following: +# # [[directive: implement, title: charge, id:impl_charge, link: req_charge]]]] +# # The equivalent need object in rst is: +# # .. implement:: implement charge +# # :id: impl_charge +# # :link: req_charge +# "oneline_comment_style": { +# "start": "[[", +# "end": "]]", +# "option_separator": ",", +# "key_value_separator": ":", +# ## What's the point if the comment has no readability? +# # "default-need-type": "implements", +# # "structure": [ +# # "id", +# # "link-type", +# # "link-id", +# # "title", +# # ], +# }, +# "multiline_comment_style": { +# "line-start-char": "*", +# "start": "[[[", +# "end": "]]]", +# }, +# } +# } diff --git a/tests/data/sphinx/index.rst b/tests/data/sphinx/index.rst new file mode 100644 index 0000000..e0d730b --- /dev/null +++ b/tests/data/sphinx/index.rst @@ -0,0 +1,11 @@ +.. src-trace:: dcdc_supercharge + :project: dcdc + :file: supercharge.cpp + +.. src-trace:: dcdc_charge + :project: dcdc + :directory: ./charge + +.. src-trace:: dcdc_discharge + :project: dcdc + :directory: ./discharge diff --git a/tests/data/sphinx/make.bat b/tests/data/sphinx/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/tests/data/sphinx/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/tests/data/sphinx/src_trace.toml b/tests/data/sphinx/src_trace.toml new file mode 100644 index 0000000..b117ce5 --- /dev/null +++ b/tests/data/sphinx/src_trace.toml @@ -0,0 +1,28 @@ +[src_trace] +set_local_url = true +local_url_field = "local-url" +set_remote_url = true +remote_url_field = "remote-url" +debug_measurement = true + +[src_trace.projects.dcdc] +comment_type = "cpp" +src_dir = "../dcdc" +remote_url_pattern = "https://github.com/useblocks/ubtrace/blob/{commit}/{path}#L{line}" +exclude = ["dcdc/src/ubt/ubt.cpp"] +include = ["**/*.cpp", "**/*.hpp"] +gitignore = true + +[src_trace.projects.dcdc.oneline_comment_style] +start_sequence = "[[" +end_sequence = "]]" # default is newline character +field_split_char = "," +needs_fields = [ + { "name" = "id" }, + { "name" = "title" }, + { "name" = "type", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, + { "name" = "status", "default" = "open" }, + { "name" = "priority", "default" = "low" }, +] diff --git a/tests/doc_test/recursive_dirs/conf.py b/tests/doc_test/recursive_dirs/conf.py new file mode 100644 index 0000000..5a3e1f8 --- /dev/null +++ b/tests/doc_test/recursive_dirs/conf.py @@ -0,0 +1,66 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "test_parse" +copyright = "2025, useblocks" +author = "team useblocks" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +# html_static_path = ["_static"] + +extensions = ["sphinx_needs", "ubt_source_tracing"] + +needs_types = [ + { + "directive": "story", + "title": "User Story", + "prefix": "US_", + "color": "#BFD8D2", + "style": "node", + }, + { + "directive": "spec", + "title": "Specification", + "prefix": "SP_", + "color": "#FEDCD2", + "style": "node", + }, + { + "directive": "implement", + "title": "Implementation", + "prefix": "IM_", + "color": "#DF744A", + "style": "node", + }, + { + "directive": "impl", + "title": "Impl", + "prefix": "IMPL_", + "color": "#DFd44A", + "style": "node", + }, + { + "directive": "test", + "title": "Test Case", + "prefix": "TC_", + "color": "#DCB239", + "style": "node", + }, +] + +src_trace_config_from_toml = "src_trace.toml" diff --git a/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_1.cpp b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_1.cpp new file mode 100644 index 0000000..fa31b64 --- /dev/null +++ b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_1.cpp @@ -0,0 +1,8 @@ + +#include + +// [[ directive:implement, title: implement req 1, id: IMPL_1, link: REQ_1 ]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} diff --git a/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_2.cpp b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_2.cpp new file mode 100644 index 0000000..5c2d046 --- /dev/null +++ b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_2.cpp @@ -0,0 +1,8 @@ + +#include + +// [[ directive:implement, title: implement req 2, id: IMPL_2, link: REQ_2 ]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} diff --git a/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_3.cpp b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_3.cpp new file mode 100644 index 0000000..9bd7c34 --- /dev/null +++ b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_3.cpp @@ -0,0 +1,8 @@ + +#include + +// [[ directive:implement, title: implement req 3, id: IMPL_3, link: REQ_3 ]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} diff --git a/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_lv4/dummy_4.cpp b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_lv4/dummy_4.cpp new file mode 100644 index 0000000..c60debf --- /dev/null +++ b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_lv4/dummy_4.cpp @@ -0,0 +1,8 @@ + +#include + +// [[ directive:implement, title: implement req 4, id: IMPL_4, link: REQ_4 ]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} diff --git a/tests/doc_test/recursive_dirs/index.rst b/tests/doc_test/recursive_dirs/index.rst new file mode 100644 index 0000000..9a1dc96 --- /dev/null +++ b/tests/doc_test/recursive_dirs/index.rst @@ -0,0 +1,2 @@ +.. src-trace:: dummy src + :project: dummy_src diff --git a/tests/doc_test/recursive_dirs/src_trace.toml b/tests/doc_test/recursive_dirs/src_trace.toml new file mode 100644 index 0000000..da1d164 --- /dev/null +++ b/tests/doc_test/recursive_dirs/src_trace.toml @@ -0,0 +1,26 @@ +[src_trace] +set_local_url = true +local_url_field = "local-url" +set_remote_url = true +remote_url_field = "remote-url" +debug_measurement = true + +[src_trace.projects.dummy_src] +comment_type = "cpp" +src_dir = "./dummy_src_lv1" +remote_url_pattern = "https://github.com/useblocks/ubtrace/blob/{commit}/{path}#L{line}" +exclude = ["dcdc/src/ubt/ubt.cpp"] +include = ["**/*.cpp", "**/*.hpp"] +gitignore = true + +[src_trace.projects.dummy_src.oneline_comment_style] +start_sequence = "[[" +end_sequence = "]]" # default is newline character +field_split_char = "," +needs_fields = [ + { "name" = "id", "type" = "str" }, + { "name" = "title", "type" = "str" }, + { "name" = "type", "type" = "str", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, +] diff --git a/tests/test_cmd.py b/tests/test_cmd.py new file mode 100644 index 0000000..fc7711d --- /dev/null +++ b/tests/test_cmd.py @@ -0,0 +1,144 @@ +from pathlib import Path + +import pytest +from typer.testing import CliRunner +from ubt_source_tracing.cmd import app + +from .conftest import ( + BASIC_VDOC_TOML, + DEFAULT_VDOC_TOML, + RECURSIVE_DIR_VDOC_TOML, + SRC_TRACE_TOML, + TEST_DIR, +) + +runner = CliRunner() + + +@pytest.mark.parametrize( + ("options", "stdout"), + [ + ( + ["discover", str(TEST_DIR / "data" / "dcdc"), "--no-gitignore"], + "5 files discovered", + ), + ( + ["discover", str(TEST_DIR / "data" / "dcdc"), "--gitignore"], + "4 files discovered", + ), + ], +) +def test_discover(options, stdout): + result = runner.invoke(app, options) + assert result.exit_code == 0 + assert stdout in result.stdout + + +@pytest.mark.parametrize( + ("options", "lines"), + [ + ( + [ + "vdoc", + "--config", + SRC_TRACE_TOML, + "--project", + "dcdc", + ], + [ + "The virtual documents are generated:", + Path("charge") / "demo_1.json", + Path("charge") / "demo_2.json", + Path("discharge") / "demo_3.json", + Path("supercharge.json"), + "The cached files are:", + TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", + TEST_DIR / "data" / "dcdc" / "charge" / "demo_2.cpp", + TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", + TEST_DIR / "data" / "dcdc" / "supercharge.cpp", + ], + ), + ( + [ + "vdoc", + "--config", + BASIC_VDOC_TOML, + ], + [ + "The virtual documents are generated:", + Path("basic_oneliners.json"), + "The cached files are:", + TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", + ], + ), + ( + [ + "vdoc", + "--config", + DEFAULT_VDOC_TOML, + ], + [ + "The virtual documents are generated:", + Path("default_oneliners.json"), + "The cached files are:", + TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", + ], + ), + ( + ["vdoc", "--config", RECURSIVE_DIR_VDOC_TOML, "--project", "dummy_src"], + [ + "The virtual documents are generated:", + Path("dummy_1.json"), + Path("dummy_lv2") / "dummy_2.json", + Path("dummy_lv2") / "dummy_lv3" / "dummy_3.json", + Path("dummy_lv2") / "dummy_lv3" / "dummy_lv4" / "dummy_4.json", + "The cached files are:", + TEST_DIR + / "doc_test" + / "recursive_dirs" + / "dummy_src_lv1" + / "dummy_1.cpp", + TEST_DIR + / "doc_test" + / "recursive_dirs" + / "dummy_src_lv1" + / "dummy_lv2" + / "dummy_2.cpp", + TEST_DIR + / "doc_test" + / "recursive_dirs" + / "dummy_src_lv1" + / "dummy_lv2" + / "dummy_lv3" + / "dummy_3.cpp", + TEST_DIR + / "doc_test" + / "recursive_dirs" + / "dummy_src_lv1" + / "dummy_lv2" + / "dummy_lv3" + / "dummy_lv4" + / "dummy_4.cpp", + ], + ), + ], +) +def test_vdoc(options, lines, tmp_path): + options.append("--output-dir") + options.append(tmp_path) + for i in range(len(lines)): + if lines[i] == "The virtual documents are generated:": + continue + if lines[i] == "The cached files are:": + break + lines[i] = tmp_path / lines[i] + + lines = [str(line) for line in lines] + + result = runner.invoke( + app, + options, + ) + + assert result.exit_code == 0 + assert result.stdout.splitlines() == lines diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py new file mode 100644 index 0000000..0f322c5 --- /dev/null +++ b/tests/test_source_discover.py @@ -0,0 +1,38 @@ +from pathlib import Path + +from ubt_source_tracing.source_discover import SourceDiscover + + +def test_source_discover_all_files(source_directory: Path): + source_discover = SourceDiscover(source_directory, gitignore=False) + assert len(source_discover.source_paths) == 5 + + +def test_source_discover_gitignore(source_directory: Path): + source_discover = SourceDiscover(source_directory, gitignore=True) + assert len(source_discover.source_paths) == 4 + + +def test_source_discover_includes(source_directory: Path): + source_discover = SourceDiscover( + source_directory, + gitignore=True, + excludes=["charge/*.cpp"], + includes=["**/*.cpp"], + ) + assert len(source_discover.source_paths) == 5 + + +def test_source_discover_excludes(source_directory: Path): + source_discover = SourceDiscover( + source_directory, gitignore=True, excludes=["charge/*.cpp"] + ) + assert len(source_discover.source_paths) == 3 + + +def test_source_discover_type(source_directory: Path): + source_discover = SourceDiscover( + source_directory, gitignore=False, file_types=["cpp"] + ) + assert len(source_discover.source_paths) == 4 + assert all(path.suffix == ".cpp" for path in source_discover.source_paths) diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py new file mode 100644 index 0000000..11053f0 --- /dev/null +++ b/tests/test_src_trace.py @@ -0,0 +1,48 @@ +from collections.abc import Callable +from pathlib import Path +import shutil + +import pytest +from sphinx.testing.util import SphinxTestApp + + +@pytest.mark.parametrize( + ("sphinx_project", "source_code"), + [ + (Path("data") / "sphinx", Path("data") / "dcdc"), + ( + Path("doc_test") / "recursive_dirs", + Path("doc_test") / "recursive_dirs" / "dummy_src_lv1", + ), + ], +) +def test_build_html( + tmpdir: Path, + make_app: Callable[..., SphinxTestApp], + sphinx_project, + source_code, + snapshot_doctree, +): + this_file_dir = Path(__file__).parent + + sphinx_src_dir = tmpdir / sphinx_project + shutil.copytree( + this_file_dir / sphinx_project, + sphinx_src_dir, + dirs_exist_ok=True, + ) + shutil.copytree( + this_file_dir / source_code, + tmpdir / source_code, + dirs_exist_ok=True, + ) + + app: SphinxTestApp = make_app( + srcdir=Path(sphinx_src_dir), + freshenv=True, + ) + app.build() + html = Path(app.outdir, "index.html").read_text() + + assert html + assert app.env.get_doctree("index") == snapshot_doctree diff --git a/tests/test_virtual_docs.py b/tests/test_virtual_docs.py new file mode 100644 index 0000000..63b1d6b --- /dev/null +++ b/tests/test_virtual_docs.py @@ -0,0 +1,497 @@ +import os + +import pytest +from ubt_source_tracing.virtual_docs.config import ESCAPE, OneLineCommentStyle +from ubt_source_tracing.virtual_docs.utils import ( + OnelineParserInvalidWarning, + WarningSubTypeEnum, + oneline_parser, +) +from ubt_source_tracing.virtual_docs.virtual_docs import VirtualDocs + +from .conftest import TEST_DIR + +ONELINE_COMMENT_STYLE = OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "id"}, + {"name": "title"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + {"name": "status", "default": "open"}, + {"name": "priority", "default": "low"}, + ], +) + +ONELINE_COMMENT_STYLE_DEFAULT = OneLineCommentStyle() + + +@pytest.mark.parametrize( + "oneline_config, result", + [ + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[]", "default": []}, # wrong type + ], + ), + [ + "Schema validation error in need_fields 'links': 'list[]' is not one of ['str', 'list[str]']" + ], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": 123}, # int is invalid + {"name": "links", "type": "list[str]", "default": []}, + ], + ), + [ + "Schema validation error in need_fields 'type': 123 is not of type 'string'" + ], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title", "qwe": "qwe"}, # invalid qwe filed + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + ), + [ + "Schema validation error in need_fields 'title': Additional properties are not allowed ('qwe' was unexpected)" + ], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + { + "name": "type", + "type: ": "list[str]", + "default": "impl", + }, # wring combination of type and default + {"name": "links", "type": "list[str]", "default": []}, + ], + ), + [ + "Schema validation error in need_fields 'type': Additional properties are not allowed ('type: ' was unexpected)" + ], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "id"} # "title" and "type" are not given + ], + ), + ["Missing required fields: ['title', 'type']"], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "id"}, + {"name": "id"}, # duplicate + ], + ), + [ + "Missing required fields: ['title', 'type']", + "Field 'id' is defined multiple times.", + ], + ), + ( + OneLineCommentStyle( + start_sequence=1234, # wrong type + end_sequence=5678, + field_split_char=2222, + needs_fields=[ + {"name": "id"}, + ], + ), + [ + "Schema validation error in field 'field_split_char': 2222 is not of type 'string'", + "Schema validation error in field 'end_sequence': 5678 is not of type 'string'", + "Schema validation error in field 'start_sequence': 1234 is not of type 'string'", + "Missing required fields: ['title', 'type']", + ], + ), + ], +) +def test_schema_validator_negative(oneline_config, result): + errors = oneline_config.check_fields_configuration() + assert errors.sort() == result.sort() + + +@pytest.mark.parametrize( + "oneline_config", + [ + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + ), + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, # minimum need_fields config + {"name": "type"}, + ], + ), + OneLineCommentStyle( + needs_fields=[ # minimum config + {"name": "title"}, + {"name": "type"}, + ], + ), + ], +) +def test_schema_validator_positive(oneline_config): + assert len(oneline_config.check_fields_configuration()) == 0 + + +@pytest.mark.parametrize( + "oneline, result", + [ + ( + "[[IMPL_1, title 1]]", + { + "id": "IMPL_1", + "title": "title 1", + "type": "impl", + "links": [], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_2, title 2, impl, [], closed]]", + { + "id": "IMPL_2", + "title": "title 2", + "type": "impl", + "links": [], + "status": "closed", + "priority": "low", + }, + ), + ( + "[[IMPL_3, title\, 3, impl, [], closed]]", + { + "id": "IMPL_3", + "title": "title, 3", + "type": "impl", + "links": [], + "status": "closed", + "priority": "low", + }, + ), + ( + "[[IMPL_5, title 5, impl, [SPEC_1, SPEC_2], open]]", + { + "id": "IMPL_5", + "title": "title 5", + "type": "impl", + "links": ["SPEC_1", "SPEC_2"], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_7, Function has a, in the title]]", + { + "id": "IMPL_7", + "title": "Function has a", + "type": "in the title", + "links": [], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_8, [Title starts with a bracket], impl]]", + { + "id": "IMPL_8", + "title": "[Title starts with a bracket]", + "type": "impl", + "links": [], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_9, Function Baz, impl, [SPEC_1, SPEC_2[text], SPEC_3], open]]", + { + "id": "IMPL_9", + "title": "Function Baz", + "type": "impl", + "links": ["SPEC_1", "SPEC_2[text"], + "status": "SPEC_3]", + "priority": "open", + }, + ), + ( + "[[IMPL_10, title 10, impl, [SPEC_1], open]]", + { + "id": "IMPL_10", + "title": "title 10", + "type": "impl", + "links": ["SPEC_1"], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_11, title 11, impl, [SPEC\,_1], open]]", + { + "id": "IMPL_11", + "title": "title 11", + "type": "impl", + "links": ["SPEC,_1"], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_12, title 12, impl, [\[SPEC\,_1\]], open]]", + { + "id": "IMPL_12", + "title": "title 12", + "type": "impl", + "links": ["[SPEC,_1]"], + "status": "open", + "priority": "low", + }, + ), + ], +) +def test_oneline_parser_custom_config_positive(oneline: str, result): + assert oneline_parser(oneline, ONELINE_COMMENT_STYLE) == result + + +@pytest.mark.parametrize( + "oneline, result", + [ + ( + f"@title 1, IMPL_1 {os.linesep}", + { + "title": "title 1", + "id": "IMPL_1", + "type": "impl", + "links": [], + }, + ), + ], +) +def test_oneline_parser_default_config_positive(oneline: str, result): + assert oneline_parser(oneline, ONELINE_COMMENT_STYLE_DEFAULT) == result + + +@pytest.mark.parametrize( + "oneline, result", + [ + ( + f"[[IMPL_4, title{ESCAPE}{ESCAPE}, 4, impl, [], closed]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.missing_square_brackets, + msg="Field links with 'type': 'list[str]' must be given with '[]' brackets", + ), + ), + ( + "[[IMPL_2, Function Bar, impl, [SPEC_1, SPEC_2, open]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.missing_square_brackets, + msg="Field links with 'type': 'list[str]' must be given with '[]' brackets", + ), + ), + ( + "[[IMPL_13, title 13, impl, 13[\[SPEC\,_1\]], open]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.not_start_or_end_with_square_brackets, + msg="Field links with 'type': 'list[str]' must start with '[' and end with ']'", + ), + ), + ( + "[[IMPL_14, title 13, impl, 13[\[SPEC\,_1\]], open, low, high]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.too_many_fields, + msg="7 given fields. They shall be less than 6", + ), + ), + ( + "[[IMPL_15]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.too_few_fields, + msg="1 given fields. They shall be more than 2", + ), + ), + ( + f"[[IMPL_16]]{os.linesep}, title 16]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.newline_in_field, + msg="Field id has newline character. It is not allowed", + ), + ), + ], +) +def test_oneline_parser_custom_config_negative( + oneline: str, result: OnelineParserInvalidWarning +): + res = oneline_parser(oneline, ONELINE_COMMENT_STYLE) + assert res == result + + +@pytest.mark.parametrize( + "oneline, result", + [ + ( + f"@title 17]]{os.linesep}, IMPL_17 {os.linesep}", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.newline_in_field, + msg="Field title has newline character. It is not allowed", + ), + ), + ( + f"@title 17]], IMPL_17, impl, [SPEC_3, SPEC_4{os.linesep} ] {os.linesep}", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.newline_in_field, + msg="Field links has newline character. It is not allowed", + ), + ), + ], +) +def test_oneline_parser_default_config_negative(oneline: str, result): + assert oneline_parser(oneline, ONELINE_COMMENT_STYLE_DEFAULT) == result + + +@pytest.mark.parametrize( + "src_dir, src_paths , oneline_comment_style, result", + [ + ( + TEST_DIR / "data" / "dcdc", + [ + TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", + TEST_DIR / "data" / "dcdc" / "charge" / "demo_2.cpp", + TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", + TEST_DIR / "data" / "dcdc" / "supercharge.cpp", + ], + ONELINE_COMMENT_STYLE, + { + "num_virtual_docs": 4, + "num_src_files": 4, + "num_uncached_files": 4, + "num_cached_files": 0, + "num_valid_comments": 10, + "num_oneline_warnings": 2, + }, + ), + ( + TEST_DIR / "data" / "oneline_comment_basic", + [ + TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", + ], + ONELINE_COMMENT_STYLE, + { + "num_virtual_docs": 1, + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_valid_comments": 8, + "num_oneline_warnings": 0, + "warnings_path_exists": True, + }, + ), + ( + TEST_DIR / "data" / "oneline_comment_default", + [ + TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", + ], + ONELINE_COMMENT_STYLE_DEFAULT, + { + "num_virtual_docs": 1, + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_valid_comments": 4, + "num_oneline_warnings": 1, + "warnings_path_exists": True, + }, + ), + ], +) +def test_virtual_docs(tmp_path, src_dir, src_paths, oneline_comment_style, result): + virtual_docs = VirtualDocs(src_paths, src_dir, tmp_path, oneline_comment_style) + virtual_docs.collect() + + assert len(virtual_docs.virtual_docs) == result["num_virtual_docs"] + assert len(virtual_docs.src_files) == result["num_src_files"] + assert len(virtual_docs.cache.uncached_files) == result["num_uncached_files"] + assert len(virtual_docs.cache.cached_files) == result["num_cached_files"] + assert len(virtual_docs.oneline_warnings) == result["num_oneline_warnings"] + assert virtual_docs.warnings_path.exists() + + loaded_warnings = VirtualDocs.load_warnings(tmp_path) + + cnt_comments = 0 + for virtual_doc in virtual_docs.virtual_docs: + cnt_comments += len(virtual_doc.comments) + assert cnt_comments == result["num_valid_comments"] + + # generate virtual documents + virtual_docs.dump_virtual_docs() + for src_file in src_paths: + assert (tmp_path / src_file.with_suffix(".json").relative_to(src_dir)).exists() + + # cache + virtual_docs.cache.update_cache() + assert len(virtual_docs.cache.cached_files) == result["num_uncached_files"] + assert len(virtual_docs.cache.uncached_files) == result["num_cached_files"] + cache_file = tmp_path / "ubt_cache.json" + assert cache_file.exists() + + # save the current virtual documents + saved_virtual_docs = virtual_docs.virtual_docs + + # use cache + del virtual_docs + virtual_docs = VirtualDocs(src_paths, src_dir, tmp_path, oneline_comment_style) + virtual_docs.collect() + assert len(virtual_docs.cache.cached_files) == result["num_uncached_files"] + assert len(virtual_docs.cache.uncached_files) == result["num_cached_files"] + cache_file = tmp_path / "ubt_cache.json" + assert cache_file.exists() + assert VirtualDocs.load_warnings(tmp_path) == loaded_warnings + assert virtual_docs.virtual_docs == saved_virtual_docs From f347c6425357ba8a098f452bb547c13dbc25da56 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 11 Jun 2025 17:35:46 +0200 Subject: [PATCH 02/54] refactor name --- conftest.py | 19 +++++++++++++++++++ pyproject.toml | 2 +- src/sphinx_codelinks/README.md | 3 ++- src/sphinx_codelinks/__init__.py | 2 +- src/sphinx_codelinks/cmd.py | 10 +++++----- .../sphinx_extension/config.py | 3 ++- .../sphinx_extension/directives/src_trace.py | 13 +++++++------ .../sphinx_extension/source_tracing.py | 13 +++++++------ src/sphinx_codelinks/virtual_docs/utils.py | 2 +- .../virtual_docs/virtual_docs.py | 11 ++++------- tests/data/sphinx/conf.py | 2 +- tests/doc_test/recursive_dirs/conf.py | 2 +- tests/test_cmd.py | 3 ++- tests/test_source_discover.py | 2 +- tests/test_virtual_docs.py | 7 ++++--- 15 files changed, 58 insertions(+), 36 deletions(-) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..c58dfaa --- /dev/null +++ b/conftest.py @@ -0,0 +1,19 @@ +""" +Global pytest conftest.py. + +This is needed due to: + +pytest test discovery error for workspace: /home/marco/ub/ubtrace + Failed: Defining 'pytest_plugins' in a non-top-level conftest is no longer supported: +It affects the entire test suite instead of just below the conftest as expected. + /home/marco/ub/ubtrace/python/ubt_connect_core/tests/conftest.py +Please move it to a top level conftest file at the rootdir: + /home/marco/ub/ubtrace +For more information, visit: + https://docs.pytest.org/en/stable/deprecations.html#pytest-plugins-in-non-top-level-conftest-files + +See also the root README.md. +""" + +# Makes make_app avaialble +pytest_plugins = ("sphinx.testing.fixtures",) diff --git a/pyproject.toml b/pyproject.toml index 5aa5db8..64a5029 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -156,7 +156,7 @@ module = ["ublicense.*"] disallow_any_expr = false [[tool.mypy.overrides]] -module = "ubt_source_tracing.*" +module = "sphinx_codelinks.*" disallow_any_unimported = false disallow_untyped_defs = false disallow_any_expr = false diff --git a/src/sphinx_codelinks/README.md b/src/sphinx_codelinks/README.md index 771eb37..db188d8 100644 --- a/src/sphinx_codelinks/README.md +++ b/src/sphinx_codelinks/README.md @@ -17,9 +17,10 @@ The project consists of the following three components: - Source Tracing: Sphinx extension to represent the collected the needs in the documentation `Source Discovery` and `Virtual Docs` can be used as `APIs` or `CLI tools`. -The detail usages can be found in the [test cases](./python/ubt_source_tracing/tests). +The detail usages can be found in the [test cases](./python/sphinx_codelinks/tests). The library is built to be + - ⚡ fast for large code bases and - 📃 support a multitude of languages. diff --git a/src/sphinx_codelinks/__init__.py b/src/sphinx_codelinks/__init__.py index 8118e2a..fa49be7 100644 --- a/src/sphinx_codelinks/__init__.py +++ b/src/sphinx_codelinks/__init__.py @@ -1,6 +1,6 @@ """ubTrace source code analyzer""" -from ubt_source_tracing.sphinx_extension.source_tracing import setup +from sphinx_codelinks.sphinx_extension.source_tracing import setup __version__ = "0.1.0" diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 4700813..beb5f5f 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -52,7 +52,7 @@ def discover( ] = None, ) -> None: """Discover the filepaths from the given root directory.""" - from ubt_source_tracing.source_discover import SourceDiscover + from sphinx_codelinks.source_discover import SourceDiscover source_discover = SourceDiscover( root_dir=root_dir, @@ -92,7 +92,7 @@ def vdoc( ) -> None: """Generate virtual documents for caching and extract the oneline comments.""" - from ubt_source_tracing.virtual_docs.config import OneLineCommentStyle + from sphinx_codelinks.virtual_docs.config import OneLineCommentStyle data = load_config_from_toml(config, project) # src_dir = Path(data["src_dir"]) @@ -116,8 +116,8 @@ def vdoc( raise typer.BadParameter( f"Invalid oneline comment style configuration: {linesep.join(errors)}" ) - from ubt_source_tracing.source_discover import SourceDiscover - from ubt_source_tracing.virtual_docs.utils import get_file_types + from sphinx_codelinks.source_discover import SourceDiscover + from sphinx_codelinks.virtual_docs.utils import get_file_types file_types = get_file_types(comment_type) @@ -129,7 +129,7 @@ def vdoc( gitignore=gitignore, ) - from ubt_source_tracing.virtual_docs.virtual_docs import VirtualDocs + from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs virtual_docs = VirtualDocs( src_files=source_discover.source_paths, diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index 00e0549..e835860 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -3,7 +3,8 @@ from sphinx.application import Sphinx from sphinx.config import Config as _SphinxConfig -from ubt_source_tracing.virtual_docs.config import ( + +from sphinx_codelinks.virtual_docs.config import ( OneLineCommentStyle, OneLineCommentStyleType, ) diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py index 977ebee..b62afc7 100644 --- a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -10,17 +10,18 @@ from sphinx.util.docutils import SphinxDirective from sphinx_needs.api import add_need # type: ignore[import-untyped] from sphinx_needs.utils import add_doc # type: ignore[import-untyped] -from ubt_source_tracing.source_discover import SourceDiscover -from ubt_source_tracing.sphinx_extension.config import ( + +from sphinx_codelinks.source_discover import SourceDiscover +from sphinx_codelinks.sphinx_extension.config import ( SRC_TRACE_CACHE, SrcTraceProjectConfigType, SrcTraceSphinxConfig, file_lineno_href, ) -from ubt_source_tracing.sphinx_extension.debug import measure_time -from ubt_source_tracing.virtual_docs.ubt_models import UBTComment -from ubt_source_tracing.virtual_docs.utils import get_file_types -from ubt_source_tracing.virtual_docs.virtual_docs import VirtualDocs +from sphinx_codelinks.sphinx_extension.debug import measure_time +from sphinx_codelinks.virtual_docs.ubt_models import UBTComment +from sphinx_codelinks.virtual_docs.utils import get_file_types +from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs sphinx_version = sphinx.__version__ diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py index b0df635..c99ae7e 100644 --- a/src/sphinx_codelinks/sphinx_extension/source_tracing.py +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -14,22 +14,23 @@ add_extra_option, add_need_type, ) -from ubt_source_tracing.sphinx_extension import debug -from ubt_source_tracing.sphinx_extension.config import ( + +from sphinx_codelinks.sphinx_extension import debug +from sphinx_codelinks.sphinx_extension.config import ( SRC_TRACE_CACHE, SrcTraceSphinxConfig, file_lineno_href, ) -from ubt_source_tracing.sphinx_extension.directives.src_trace import ( +from sphinx_codelinks.sphinx_extension.directives.src_trace import ( SourceTracing, SourceTracingDirective, ) -from ubt_source_tracing.sphinx_extension.html_wrapper import html_wrapper -from ubt_source_tracing.virtual_docs.config import ( +from sphinx_codelinks.sphinx_extension.html_wrapper import html_wrapper +from sphinx_codelinks.virtual_docs.config import ( OneLineCommentStyle, OneLineCommentStyleType, ) -from ubt_source_tracing.virtual_docs.virtual_docs import VirtualDocs +from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs logger = logging.getLogger(__name__) diff --git a/src/sphinx_codelinks/virtual_docs/utils.py b/src/sphinx_codelinks/virtual_docs/utils.py index 3fb8425..c30eda1 100644 --- a/src/sphinx_codelinks/virtual_docs/utils.py +++ b/src/sphinx_codelinks/virtual_docs/utils.py @@ -3,7 +3,7 @@ import logging import os -from ubt_source_tracing.virtual_docs.config import ( +from sphinx_codelinks.virtual_docs.config import ( ESCAPE, SUPPORTED_COMMENT_TYPES, OneLineCommentStyle, diff --git a/src/sphinx_codelinks/virtual_docs/virtual_docs.py b/src/sphinx_codelinks/virtual_docs/virtual_docs.py index 6aec064..23cf7c0 100644 --- a/src/sphinx_codelinks/virtual_docs/virtual_docs.py +++ b/src/sphinx_codelinks/virtual_docs/virtual_docs.py @@ -5,16 +5,13 @@ from pathlib import Path from comment_parser.parsers.common import Comment -from ubt_source_tracing.virtual_docs.config import ( + +from sphinx_codelinks.virtual_docs.config import ( SUPPORTED_COMMENT_TYPES, OneLineCommentStyle, ) -from ubt_source_tracing.virtual_docs.ubt_models import ( - UBTCache, - UBTComment, - UBTSourceFile, -) -from ubt_source_tracing.virtual_docs.utils import ( +from sphinx_codelinks.virtual_docs.ubt_models import UBTCache, UBTComment, UBTSourceFile +from sphinx_codelinks.virtual_docs.utils import ( OnelineParserInvalidWarning, oneline_parser, ) diff --git a/tests/data/sphinx/conf.py b/tests/data/sphinx/conf.py index 9918d4f..8b2c00e 100644 --- a/tests/data/sphinx/conf.py +++ b/tests/data/sphinx/conf.py @@ -23,7 +23,7 @@ html_theme = "furo" # html_static_path = ["_static"] -extensions = ["sphinx_needs", "ubt_source_tracing"] +extensions = ["sphinx_needs", "sphinx_codelinks"] needs_types = [ { diff --git a/tests/doc_test/recursive_dirs/conf.py b/tests/doc_test/recursive_dirs/conf.py index 5a3e1f8..2c6a754 100644 --- a/tests/doc_test/recursive_dirs/conf.py +++ b/tests/doc_test/recursive_dirs/conf.py @@ -23,7 +23,7 @@ html_theme = "alabaster" # html_static_path = ["_static"] -extensions = ["sphinx_needs", "ubt_source_tracing"] +extensions = ["sphinx_needs", "sphinx_codelinks"] needs_types = [ { diff --git a/tests/test_cmd.py b/tests/test_cmd.py index fc7711d..411e1ab 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -2,7 +2,8 @@ import pytest from typer.testing import CliRunner -from ubt_source_tracing.cmd import app + +from sphinx_codelinks.cmd import app from .conftest import ( BASIC_VDOC_TOML, diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index 0f322c5..3e99612 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -1,6 +1,6 @@ from pathlib import Path -from ubt_source_tracing.source_discover import SourceDiscover +from sphinx_codelinks.source_discover import SourceDiscover def test_source_discover_all_files(source_directory: Path): diff --git a/tests/test_virtual_docs.py b/tests/test_virtual_docs.py index 63b1d6b..591e89d 100644 --- a/tests/test_virtual_docs.py +++ b/tests/test_virtual_docs.py @@ -1,13 +1,14 @@ import os import pytest -from ubt_source_tracing.virtual_docs.config import ESCAPE, OneLineCommentStyle -from ubt_source_tracing.virtual_docs.utils import ( + +from sphinx_codelinks.virtual_docs.config import ESCAPE, OneLineCommentStyle +from sphinx_codelinks.virtual_docs.utils import ( OnelineParserInvalidWarning, WarningSubTypeEnum, oneline_parser, ) -from ubt_source_tracing.virtual_docs.virtual_docs import VirtualDocs +from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs from .conftest import TEST_DIR From 482cba0be97ba208ffc825800c12d4dcc3dbfdbf Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 11 Jun 2025 18:09:08 +0200 Subject: [PATCH 03/54] add ci and update ignore for mypy --- .github/workflows/ci.yml | 121 ++++++++++++++++++ .gitignore | 3 + .python-version | 1 + pyproject.toml | 40 +++--- src/sphinx_codelinks/cmd.py | 2 +- .../sphinx_extension/config.py | 4 +- .../sphinx_extension/debug.py | 16 +-- .../sphinx_extension/directives/src_trace.py | 2 +- .../sphinx_extension/html_wrapper.py | 6 +- .../sphinx_extension/source_tracing.py | 4 +- src/sphinx_codelinks/virtual_docs/config.py | 6 +- .../virtual_docs/virtual_docs.py | 4 +- 12 files changed, 164 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .python-version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a0c727b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,121 @@ +name: ci + +on: + push: + branches: [main] + tags: + - "v[0-9]+.[0-9]+.[0-9]+*" + pull_request: + types: [closed, labeled, reopened, unlabeled, synchronize, opened] + +concurrency: + # For PRs, cancel in progress runs, if a new commit is pushed + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# These permissions are needed to interact with GitHub's OIDC Token endpoint. +permissions: + id-token: write + contents: read + +jobs: + pre-commit: + name: Pre-commit + runs-on: [self-hosted, linux, x64] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version-file: ".python-version" + - run: python -m pip install pre-commit pre-commit-uv + # - uses: pre-commit/action@v3.0.1 # note we don't use this, since it calls ations/cache, which actually takes longer than without it + - run: pre-commit run --all --show-diff-on-failure --color=always + + mypy: + name: MyPy + runs-on: [self-hosted, linux, x64] + + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup_rye + - run: rye run mypy:all + + pytest: + name: Pytest (${{ matrix.os }}-${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - os: linux + arch: x64 + - os: linux + arch: arm64 + - os: windows + arch: x64 + - os: macos + arch: arm64 + + runs-on: [self-hosted, "${{ matrix.os }}", "${{ matrix.arch }}"] + + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup_rye + - run: rye test -a + + pytest-prod: + # pytest against packages installed as they would be in production + # i.e. as wheels that may contain obfuscated code + + name: Pytest prod (${{ matrix.os }}-${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - os: linux + arch: x64 + - os: linux + arch: arm64 + - os: windows + arch: x64 + - os: macos + arch: arm64 + + runs-on: [self-hosted, "${{ matrix.os }}", "${{ matrix.arch }}"] + + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup_rye + - run: rye run pytest:prod + + docs: + name: Documentation build + runs-on: [self-hosted, linux, x64] + + steps: + - uses: actions/checkout@v4 + - name: Install graphviz + run: sudo apt-get --yes install graphviz + - uses: ./.github/actions/setup_rye + - name: Run documentation build + run: rye run docs_html + + all_good: + # This job does nothing and is only used for the branch protection + # see https://github.com/marketplace/actions/alls-green#why + + if: ${{ !cancelled() }} + + needs: + - pre-commit + - mypy + - pytest + - pytest-prod + - docs + + runs-on: [self-hosted, linux, x64] + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.gitignore b/.gitignore index fcd7543..3f63402 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ requirements-dev.lock # Sphinx build output **/_build + +# rye is the primary tool, uv is only used for on-the-fly setups +uv.lock diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..56bb660 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12.7 diff --git a/pyproject.toml b/pyproject.toml index 64a5029..f122feb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dev-dependencies = [ "moto ~= 5.0", "mypy>=1.12.1", "myst-parser>=4.0.0", + "pydantic ~= 2.9", "pip-licenses>=5.0.0", "psutil>=7.0.0", "pytest-cov>=5.0.0", @@ -50,6 +51,19 @@ dev-dependencies = [ "types-jsonschema>=4.23.0.20241208", ] +[tool.rye.scripts] +# linting and formatting +"mypy:all" = "mypy ." +"rye:lint" = "rye lint" +"rye:format" = "rye format" +"check" = { chain = ["rye:format", "rye:lint", "mypy:all"] } +# docs html +"docs_html:rm" = "rm -rf docs/_build/html" +"docs_html" = "sphinx-build -nW --keep-going -b html -T -c docs docs/source docs/_build/html" +"docs_html:clean" = { chain = ["docs_html:rm", "docs_html"] } +# pytest prod +"pytest:prod" = { cmd = "uv run --with-requirements requirements.lock --no-editable --refresh pytest tests/" } + [tool.ruff.lint] extend-select = [ # "ANN", @@ -102,20 +116,7 @@ force-sort-within-sections = true ] [tool.mypy] -exclude = [ - "^.*/tests/", - "dist/", - "docs/_build/", - "docs/conf.py", - "python/ubt_autodiscover/src/", # TODO still untyped - "_python_archive/ubt_connect_core/src/", # TODO still untyped - "_python_archive/ubt_connect_services/src/", # TODO still untyped - "python/ubt_db/src/", # TODO still untyped - "python/ubt_runtime/src/ubt_runtime/__init__.py", # generated source - "python/ubt_server/src/", # TODO still untyped - "python/ubt_sphinx/src/", # TODO still untyped - "scripts/sync_metadata.py", # TODO investigate why this fails here but not in ubcode -] +exclude = ["tests/", "dist/", "docs/_build/", "docs/conf.py"] show_error_codes = true warn_unused_ignores = true warn_redundant_casts = true @@ -140,10 +141,6 @@ mypy_path = "typings" module = ["licensing.*", "tomlkit.*"] ignore_missing_imports = true -[[tool.mypy.overrides]] -module = ["scripts.*", "build_hooks.*", "ublicense.*"] -disallow_any_expr = false - [[tool.mypy.overrides]] module = "tests.*" disallow_any_explicit = false @@ -151,20 +148,17 @@ disallow_any_unimported = false disallow_untyped_defs = false disallow_any_expr = false -[[tool.mypy.overrides]] -module = ["ublicense.*"] -disallow_any_expr = false - [[tool.mypy.overrides]] module = "sphinx_codelinks.*" disallow_any_unimported = false disallow_untyped_defs = false disallow_any_expr = false + [tool.pydantic-mypy] init_forbid_extra = true init_typed = true warn_required_dynamic_aliases = true [tool.pytest.ini_options] -testpaths = ["python"] +testpaths = ["tests"] diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index beb5f5f..8e5a2f5 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -157,7 +157,7 @@ def vdoc( typer.echo(cached_file) -def load_config_from_toml( # type: ignore[misc] +def load_config_from_toml( # type: ignore[explicit-any] toml_file: Path, project: str | None = None ) -> dict[str, Any]: try: diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index e835860..4ebe16b 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -38,12 +38,12 @@ class SrcTraceSphinxConfig: def __init__(self, config: _SphinxConfig) -> None: super().__setattr__("_config", config) - def __getattribute__(self, name: str) -> Any: # type: ignore[misc] + def __getattribute__(self, name: str) -> Any: # type: ignore[explicit-any] if name.startswith("__") or name == "_config": return super().__getattribute__(name) return getattr(super().__getattribute__("_config"), f"src_trace_{name}") - def __setattr__(self, name: str, value: Any) -> None: # type: ignore[misc] + def __setattr__(self, name: str, value: Any) -> None: # type: ignore[explicit-any] if name == "_config" and "src_trace_projects" in value: src_trace_projects: dict[str, SrcTraceProjectConfigType] = value[ "src_trace_projects" diff --git a/src/sphinx_codelinks/sphinx_extension/debug.py b/src/sphinx_codelinks/sphinx_extension/debug.py index 28a8078..cf4ef4c 100644 --- a/src/sphinx_codelinks/sphinx_extension/debug.py +++ b/src/sphinx_codelinks/sphinx_extension/debug.py @@ -18,17 +18,17 @@ from sphinx.application import Sphinx # Stores the timing results -TIME_MEASUREMENTS: dict[str, Any] = {} # type: ignore[misc] +TIME_MEASUREMENTS: dict[str, Any] = {} # type: ignore[explicit-any] EXECUTE_TIME_MEASUREMENTS = ( False # Will be used to de/activate measurements. Set during a Sphinx Event ) START_TIME = 0.0 -T = TypeVar("T", bound=Callable[..., Any]) # type: ignore[misc] +T = TypeVar("T", bound=Callable[..., Any]) # type: ignore[explicit-any] -def measure_time( # type: ignore[misc] +def measure_time( # type: ignore[explicit-any] category: str | None = None, source: str = "internal", name: str | None = None ) -> Callable[[T], T]: """ @@ -58,9 +58,9 @@ def my_cool_function(a, b,c ): :param name: Name to use for the measured. If not given, the function name is used. """ - def inner(func: T) -> T: # type: ignore[misc] + def inner(func: T) -> T: # type: ignore[explicit-any] @wraps(func) - def wrapper(*args: list[object], **kwargs: dict[object, object]) -> Any: # type: ignore[misc] + def wrapper(*args: list[object], **kwargs: dict[object, object]) -> Any: # type: ignore[explicit-any] """ Wrapper function around a given/decorated function, which cares about measurement and storing the result @@ -125,7 +125,7 @@ def wrapper(*args: list[object], **kwargs: dict[object, object]) -> Any: # type return inner -def measure_time_func( # type: ignore[misc] +def measure_time_func( # type: ignore[explicit-any] func: T, category: str | None = None, source: str = "internal", @@ -154,7 +154,7 @@ def _print_timing_results() -> None: print(f" min: {value['min']:2f} \n") -def _store_timing_results_json(app: Sphinx, build_data: dict[str, Any]) -> None: # type: ignore[misc] +def _store_timing_results_json(app: Sphinx, build_data: dict[str, Any]) -> None: # type: ignore[explicit-any] json_result_path = Path(app.outdir) / "debug_measurement.json" data = {"build": build_data, "measurements": TIME_MEASUREMENTS} @@ -163,7 +163,7 @@ def _store_timing_results_json(app: Sphinx, build_data: dict[str, Any]) -> None: print(f"Timing measurement results (JSON) stored under {json_result_path}") -def _store_timing_results_html(app: Sphinx, build_data: dict[str, Any]) -> None: # type: ignore[misc] +def _store_timing_results_html(app: Sphinx, build_data: dict[str, Any]) -> None: # type: ignore[explicit-any] jinja_env = Environment( loader=PackageLoader("sphinx_needs"), autoescape=select_autoescape() ) diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py index b62afc7..20c72d1 100644 --- a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -322,7 +322,7 @@ def render_needs( title=str( comment.resolved_marker["title"] ), # The title of the need - **cast(dict[str, Any], kwargs), # type: ignore[misc] + **cast(dict[str, Any], kwargs), # type: ignore[explicit-any] ) rendered_needs.extend(marker_needs) if local_url_field: diff --git a/src/sphinx_codelinks/sphinx_extension/html_wrapper.py b/src/sphinx_codelinks/sphinx_extension/html_wrapper.py index 2be89fc..ca85822 100644 --- a/src/sphinx_codelinks/sphinx_extension/html_wrapper.py +++ b/src/sphinx_codelinks/sphinx_extension/html_wrapper.py @@ -8,14 +8,14 @@ class LineFormatter(HtmlFormatter): # type: ignore[type-arg] - def __init__(self, lineno_href: dict[int, str], *args: Any, **kwargs: Any) -> None: # type: ignore[misc] + def __init__(self, lineno_href: dict[int, str], *args: Any, **kwargs: Any) -> None: # type: ignore[explicit-any] super().__init__(*args, **kwargs) self.lineno_href = lineno_href - def wrap(self, source: Generator[Any]) -> Generator[Any]: # type: ignore[misc] + def wrap(self, source: Generator[Any]) -> Generator[Any]: # type: ignore[explicit-any] return self._wrap_custom_lines(super().wrap(source)) # type: ignore[no-untyped-call] - def _wrap_custom_lines(self, source: Generator[Any]) -> Generator[Any]: # type: ignore[misc] + def _wrap_custom_lines(self, source: Generator[Any]) -> Generator[Any]: # type: ignore[explicit-any] lineno = 0 for is_line, line_html in source: if is_line: diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py index c99ae7e..edecca3 100644 --- a/src/sphinx_codelinks/sphinx_extension/source_tracing.py +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -35,7 +35,7 @@ logger = logging.getLogger(__name__) -def setup(app: Sphinx) -> dict[str, Any]: # type: ignore[misc] +def setup(app: Sphinx) -> dict[str, Any]: # type: ignore[explicit-any] app.add_node(SourceTracing) app.add_directive("src-trace", SourceTracingDirective) SrcTraceSphinxConfig.add_config_values(app) @@ -64,7 +64,7 @@ def builder_inited(app: Sphinx) -> None: copy_asset(custom_css, Path(app.outdir) / "_static" / "source_tracing") -def add_custom_css( # type: ignore[misc] +def add_custom_css( # type: ignore[explicit-any] app: Sphinx, pagename: str, templatename: str, diff --git a/src/sphinx_codelinks/virtual_docs/config.py b/src/sphinx_codelinks/virtual_docs/config.py index dd8bd6b..f330f16 100644 --- a/src/sphinx_codelinks/virtual_docs/config.py +++ b/src/sphinx_codelinks/virtual_docs/config.py @@ -34,7 +34,7 @@ class OneLineCommentStyleType(TypedDict): @dataclass class OneLineCommentStyle: - def __setattr__(self, name: str, value: Any) -> None: # type: ignore[misc] + def __setattr__(self, name: str, value: Any) -> None: # type: ignore[explicit-any] if name == "needs_fields": # apply default to fields self.apply_needs_field_default(value) @@ -134,10 +134,10 @@ def get_required_fields(cls, name: str) -> list[str] | None: return None @classmethod - def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[misc] + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] _field = next(_field for _field in fields(cls) if _field.name is name) if _field.metadata is not MISSING and "schema" in _field.metadata: - return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[misc] + return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[explicit-any] return None def check_schema(self) -> list[str]: diff --git a/src/sphinx_codelinks/virtual_docs/virtual_docs.py b/src/sphinx_codelinks/virtual_docs/virtual_docs.py index 23cf7c0..c9d1b51 100644 --- a/src/sphinx_codelinks/virtual_docs/virtual_docs.py +++ b/src/sphinx_codelinks/virtual_docs/virtual_docs.py @@ -4,7 +4,7 @@ import os from pathlib import Path -from comment_parser.parsers.common import Comment +from comment_parser.parsers.common import Comment # type: ignore[import-untyped] from sphinx_codelinks.virtual_docs.config import ( SUPPORTED_COMMENT_TYPES, @@ -87,7 +87,7 @@ def collect(self) -> None: for comment in ml_comments: single_lines = comment.text().splitlines() for idx, line in enumerate(single_lines): - oneline_comments.append(Comment(line, comment.line_number() + idx)) # type: ignore[call-arg] + oneline_comments.append(Comment(line, comment.line_number() + idx)) ubt_comments: list[UBTComment] = [] From 3cbde6e780789816a9278c217edaca71d487db21 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 11 Jun 2025 18:12:05 +0200 Subject: [PATCH 04/54] add action --- .github/actions/setup_rye/action.yml | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/actions/setup_rye/action.yml diff --git a/.github/actions/setup_rye/action.yml b/.github/actions/setup_rye/action.yml new file mode 100644 index 0000000..e370827 --- /dev/null +++ b/.github/actions/setup_rye/action.yml @@ -0,0 +1,31 @@ +name: Set up rye +runs: + using: 'composite' + steps: + # rye uses uv under the hood, so we need to set the cache directory correctly, based on the OS + - name: Set UV_CACHE_DIR for Linux + if: runner.os == 'Linux' + run: | + echo "UV_CACHE_DIR=/home/runner/.cache/uv" >> $GITHUB_ENV + shell: bash + - name: Set MATURIN_PEP517_ARGS for Linux + if: runner.os == 'Linux' + # make sure we always use zig, to get manylinux2014 compatible rust binaries + run: | + echo "MATURIN_PEP517_ARGS=--zig" >> $GITHUB_ENV + shell: bash + - name: Set UV_CACHE_DIR for MacOS + if: runner.os == 'macOS' + run: echo "UV_CACHE_DIR=/Users/gh-runner/Library/Caches/uv" >> $GITHUB_ENV + shell: bash + - name: Set UV_CACHE_DIR for Windows + if: runner.os == 'Windows' + run: echo "UV_CACHE_DIR=C:\\Users\\useblocks\\AppData\\Local\\uv-${{ runner.name }}" >> $env:GITHUB_ENV + shell: pwsh + # now install rye and sync the dependencies + - uses: eifinger/setup-rye@v4 + with: + version: "0.42.0" + enable-cache: false + - run: rye sync + shell: ${{ runner.os == 'Windows' && 'powershell' || 'bash' }} From 7e7deb5c36b893493a9db4151773435d28e0fdb8 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 11 Jun 2025 18:19:01 +0200 Subject: [PATCH 05/54] updated gitignore --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 3f63402..fe94ef5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,13 @@ # python generated files +.ruff_cache +.pytest_cache +.mypy_cache __pycache__/ *.py[oc] build/ dist/ wheels/ *.egg-info - -# venv .venv # lock files From 384343d98478a898d5b1f918b6a5dfd6c2aee157 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 12 Jun 2025 09:56:13 +0200 Subject: [PATCH 06/54] add docs --- docs/conf.py | 67 ++++ docs/source/_static/favicon.ico | Bin 0 -> 15406 bytes docs/source/_static/furo.css | 331 ++++++++++++++++++ .../_static/sphinx-codelinks-logo_dark.svg | 0 .../_static/sphinx-codelinks-logo_light.svg | 0 docs/source/index.rst | 61 ++++ docs/src_trace.toml | 29 ++ docs/ubproject.toml | 12 + 8 files changed, 500 insertions(+) create mode 100644 docs/conf.py create mode 100644 docs/source/_static/favicon.ico create mode 100644 docs/source/_static/furo.css rename sphinx-codelinks-logo_dark.svg => docs/source/_static/sphinx-codelinks-logo_dark.svg (100%) rename sphinx-codelinks-logo_light.svg => docs/source/_static/sphinx-codelinks-logo_light.svg (100%) create mode 100644 docs/source/index.rst create mode 100644 docs/src_trace.toml create mode 100644 docs/ubproject.toml diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..304fe0d --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,67 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +from datetime import datetime +from pathlib import Path +import tomllib + +_project_data = tomllib.loads( + (Path(__file__).parent.parent / "pyproject.toml").read_text("utf8") +)["project"] + +project = "ubtrace" +author = _project_data["authors"][0]["name"] +copyright = f"{datetime.now().year}, {author}" +version = release = _project_data["version"] + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx_needs", + "sphinx_codelinks", +] + +# exclude_patterns = [] +templates_path = ["_templates"] +show_warning_types = True + +todo_include_todos = True + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_title = "CodeLinks" +html_theme = "furo" +# original source is in ubdocs repo at docs/developer_handbook/design/files/ubcode_favicon/favicon.ico +html_favicon = "source/_static/favicon.ico" +html_static_path = ["source/_static"] + +html_theme_options = { + "sidebar_hide_name": True, + "top_of_page_buttons": ["view", "edit"], + "source_repository": "https://github.com/useblocks/sphinx-codelinks", + "source_branch": "main", + "source_directory": "docs/source/", + "light_logo": "sphinx-codelinks-logo_dark.svg", + "dark_logo": "sphinx-codelinks-logo_light.svg", +} +html_css_files = ["furo.css"] + +src_trace_config_from_toml = "./src_trace.toml" + +needs_types = [ + { + "directive": "impl", + "title": "Implementation", + "prefix": "IMPL_", + "color": "#DF744A", + "style": "node", + }, +] diff --git a/docs/source/_static/favicon.ico b/docs/source/_static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..39dce04727130a2e341bd5eefb5fa22edff54cb7 GIT binary patch literal 15406 zcmeHOd2AF_7=NYR-I=!Zp!BBgmL8Nsu@qXg?IJ-)R5Y!U5Q-Ry_=gZUH25CAmu0xPra|s zTl)6VRQkyfn!f3ww2`GOugJ1|bMIdATsadKlaB5rM^X6@AEjm&Qd(|tSa^8{@<(iS z_3y2viTk%3LPO6ca+X%=Y%uD-wsXfQ`{rpnS@z$ES+~>3IkUqeqjAy@S(c9#rAr>7 zif5JvMEUZ^kza*DbcD_y&)_L4xbL12vYdnIRQcRfRJ&%?5ER_o6-1V8X4zQVkJ1KX zXum8Uec^t_4NL>Le==AyVEZEA3*b-q?`o{^_dbK4r92bQ7SIu>IM7qd-kdQP0590NuY?uUl%9$WpI^OYw|g9NK=a~ZnImJ0_4w@@ zsIC8aRG9MfLCTywB|`oBF+jBKul27VD0})XQRt3+{MFWfjB;0u6>0Zg+uos!Tk^5ie35hUYf$a zpv=$jY77E%XL}Ouf9Brs@jK)!i*Ai2k^DUar>K7GI;=I13F3Fv|G}@7E4#^qb`x*# zenS2G>&R81-A@V5Gf?hx9m@~Du{HjfaucNm&G*Rd2xechKIzM34%jybGMbtxf8HG1 zAg|+2W&PP`&Q!k8O+0S016t31PZcYkBwj-|%0H3sU+uWG?R+nlEqjb2JqKn7{=5cV zx$Ejz?=B~9P`8j-BOZw_`PlBdAIA&Iz>%~dq^HL&Lscq&k3k8vf};gxpJELyin%k z78AM0Z~d_O$6b^&b0&$@Otbi7{gD5+V`%?lw*Ons9H8N?Z6t=7#hp1zUt|6AMeyf+ zC7n;q;_tb7hMJGGg}U7~w}H8%zZB&<=GVWVUfKpb&fCxT zoHNpgpvq@CLKHqDAuWRqb~?Xt1pQ>Uf%6-xUA(W&ZK4ZyYyf(Jc%Pr?gU*fMFc(~q Q+rpueOJ0*6Fv|n~0Nzy>!vFvP literal 0 HcmV?d00001 diff --git a/docs/source/_static/furo.css b/docs/source/_static/furo.css new file mode 100644 index 0000000..8e846ee --- /dev/null +++ b/docs/source/_static/furo.css @@ -0,0 +1,331 @@ +/* Styling for the https://github.com/pradyunsg/furo theme. */ + +:root { + --ub-color-neutral-0: #FFFFFF; + --ub-color-neutral-50: #FAFAFA; + --ub-color-neutral-100: #F5F5F5; + --ub-color-neutral-200: #EEEEEE; + --ub-color-neutral-300: #E0E0E0; + --ub-color-neutral-400: #BDBDBD; + --ub-color-neutral-500: #9E9E9E; + --ub-color-neutral-600: #757575; + --ub-color-neutral-700: #616161; + --ub-color-neutral-800: #424242; + --ub-color-neutral-900: #212121; + --ub-color-neutral-1000: #000000; +} + +/* furo light colors */ +body { + --ub-color-brand-main: #583eff; + --ub-color-brand-muted: #b7a3ff; + --ub-color-brand-opaque: rgb(88, 62, 255, 0.2); + --sn-architecture-bg: url(../architecture_bg-light.png); + + --color-brand-primary: var(--ub-color-brand-main); + + /* anchored heading title */ + --color-highlight-on-target: var(--ub-color-brand-opaque); + + /* Left ToC */ + --color-sidebar-brand-text: var(--color-foreground-primary); + --color-sidebar-caption-text: var(--ub-color-neutral-900); + --color-sidebar-link-text--top-level: var(--ub-color-neutral-800); + --color-sidebar-link-text: var(--ub-color-neutral-600); + --color-sidebar-link-text--current: var(--ub-color-brand-main); + --color-sidebar-item-background--hover: var(--ub-color-brand-opaque); + + /* Right ToC */ + --color-toc-item-text--active: var(--ub-color-brand-main); + + /* Links */ + --color-link: var(--color-content-foreground); + --color-link--hover: var(--color-content-foreground); + --color-link-underline: var(--ub-color-brand-muted); + --color-link-underline--hover: var(--ub-color-brand-main); + --color-link--visited: var(--color-content-foreground); + --color-link--visited--hover: var(--color-content-foreground); + --color-link-underline--visited: var(--ub-color-brand-muted); + --color-link-underline--visited--hover: var(--ub-color-brand-main); + + /* Admonitions */ + --color-admonition-title-background--note: var(--ub-color-brand-opaque); + --color-admonition-title--note: var(--ub-color-brand-main); + + /* Sphinx Design */ + --sd-fontsize-dropdown: var(--admonition-font-size); + --sd-fontsize-dropdown-title: var(--admonition-title-font-size); + --sd-fontweight-dropdown-title: 500; + --sd-color-card-header: var(--ub-color-brand-opaque); + --sd-color-card-border-hover: var(--ub-color-brand-opaque); +} + +/* furo dark colors */ +@media not print { + body[data-theme="dark"] { + + --ub-color-brand-main: #e4ff3e; + --ub-color-brand-muted: #b3bb00; + --ub-color-brand-opaque: rgba(228, 255, 62, 0.15); + --sn-architecture-bg: url(../architecture_bg-dark.png); + + --color-brand-primary: var(--ub-color-brand-main); + + /* anchored heading title */ + --color-highlight-on-target: var(--ub-color-brand-opaque); + + /* Left ToC */ + --color-sidebar-brand-text: var(--color-foreground-primary); + --color-sidebar-caption-text: var(--ub-color-neutral-100); + --color-sidebar-link-text--top-level: var(--ub-color-neutral-300); + --color-sidebar-link-text: var(--ub-color-neutral-500); + --color-sidebar-link-text--current: var(--ub-color-brand-main); + --color-sidebar-item-background--hover: var(--ub-color-brand-opaque); + + /* Right ToC */ + --color-toc-item-text--active: var(--ub-color-brand-main); + + /* Links */ + --color-link: var(--color-content-foreground); + --color-link--hover: var(--color-content-foreground); + --color-link-underline: var(--ub-color-brand-muted); + --color-link-underline--hover: var(--ub-color-brand-main); + --color-link--visited: var(--color-content-foreground); + --color-link--visited--hover: var(--color-content-foreground); + --color-link-underline--visited: var(--ub-color-brand-muted); + --color-link-underline--visited--hover: var(--ub-color-brand-main); + + /* Admonitions */ + --color-admonition-title-background--note: var(--ub-color-brand-opaque); + --color-admonition-title--note: var(--ub-color-brand-main); + } + + @media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) { + + --ub-color-brand-main: #e4ff3e; + --ub-color-brand-muted: #b3bb00; + --ub-color-brand-opaque: rgba(228, 255, 62, 0.15); + --sn-architecture-bg: url(../architecture_bg-dark.png); + + --color-brand-primary: var(--ub-color-brand-main); + + /* anchored heading title */ + --color-highlight-on-target: var(--ub-color-brand-opaque); + + /* Left ToC */ + --color-sidebar-brand-text: var(--color-foreground-primary); + --color-sidebar-caption-text: var(--ub-color-neutral-100); + --color-sidebar-link-text--top-level: var(--ub-color-neutral-300); + --color-sidebar-link-text: var(--ub-color-neutral-500); + --color-sidebar-link-text--current: var(--ub-color-brand-main); + --color-sidebar-item-background--hover: var(--ub-color-brand-opaque); + + /* Right ToC */ + --color-toc-item-text--active: var(--ub-color-brand-main); + + /* Links */ + --color-link: var(--color-content-foreground); + --color-link--hover: var(--color-content-foreground); + --color-link-underline: var(--ub-color-brand-muted); + --color-link-underline--hover: var(--ub-color-brand-main); + --color-link--visited: var(--color-content-foreground); + --color-link--visited--hover: var(--color-content-foreground); + --color-link-underline--visited: var(--ub-color-brand-muted); + --color-link-underline--visited--hover: var(--ub-color-brand-main); + + /* Admonitions */ + --color-admonition-title-background--note: var(--ub-color-brand-opaque); + --color-admonition-title--note: var(--ub-color-brand-main); + } + } +} + +/* sphinx-needs colors */ +/* doc config start */ +/* Note, the recommended way to set colors for furo is in the `html_theme_options` +https://pradyunsg.me/furo/customisation/#light-css-variables-dark-css-variables + +But here we are setting the colors directly in the CSS, +to make it a little easier to compare to the different themes. +*/ +body { + --color-code-background: #eeffcc; + --color-code-foreground: black; + --sn-color-need-border: #555; + --sn-color-need-row-border: hsla(232, 75%, 95%, 0.12); + --sn-color-need-bg: #eee; + --sn-color-need-bg-head: rgba(0, 0, 0, 0.1); + --sn-color-complete-bg-head: rgba(0, 0, 0, 0.1); + --sn-color-complete-bg-foot: rgba(0, 0, 0, 0.1); + --sn-color-bg-gray: #eee; + --sn-color-bg-lightgray: rgba(0, 0, 0, 0.004); + --sn-color-bg-green: #05c46b; + --sn-color-bg-red: #ff3f34; + --sn-color-bg-yellow: #ffc048; + --sn-color-bg-blue: #0fbcf9; + --sn-color-debug-btn-border: #333; + --sn-color-debug-btn-on-text: #f43333; + --sn-color-debug-btn-off-text: #096285; + --sn-color-datatable-label: var(--color-foreground-muted); + --sn-color-datatable-btn-border: var(--color-foreground-muted); +} + +@media not print { + body[data-theme="dark"] { + --color-code-background: #202020; + --color-code-foreground: #d0d0d0; + --sn-color-need-border: #aaaaaa; + --sn-color-need-row-border: hsla(52, 75%, 5%, 0.12); + --sn-color-need-bg: #111111; + --sn-color-need-bg-head: rgba(255, 255, 255, 0.1); + --sn-color-complete-bg-head: rgba(255, 255, 255, 0.1); + --sn-color-complete-bg-foot: rgba(255, 255, 255, 0.1); + --sn-color-bg-gray: #111111; + --sn-color-bg-lightgray: rgba(255, 255, 255, 0.1); + --sn-color-bg-green: #024e2a; + --sn-color-bg-red: #81201b; + --sn-color-bg-yellow: #a97c32; + --sn-color-bg-blue: #096285; + --sn-color-debug-btn-border: #888; + --sn-color-debug-btn-on-text: #ff3f34; + --sn-color-debug-btn-off-text: #0fbcf9; + } + + @media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) { + --color-code-background: #202020; + --color-code-foreground: #d0d0d0; + --sn-color-need-border: #aaaaaa; + --sn-color-need-row-border: hsla(52, 75%, 5%, 0.12); + --sn-color-need-bg: #111111; + --sn-color-need-bg-head: rgba(255, 255, 255, 0.1); + --sn-color-complete-bg-head: rgba(255, 255, 255, 0.1); + --sn-color-complete-bg-foot: rgba(255, 255, 255, 0.1); + --sn-color-bg-gray: #111111; + --sn-color-bg-lightgray: rgba(255, 255, 255, 0.1); + --sn-color-bg-green: #024e2a; + --sn-color-bg-red: #81201b; + --sn-color-bg-yellow: #a97c32; + --sn-color-bg-blue: #096285; + --sn-color-debug-btn-border: #888; + --sn-color-debug-btn-on-text: #ff3f34; + --sn-color-debug-btn-off-text: #0fbcf9; + } + } +} + +/* doc config end */ + +/* make the left ToC use the brand color for the current page */ +.sidebar-tree .current-page>.reference { + font-weight: 700; + color: var(--ub-color-brand-main) +} + +/* styling fo the icon at the top of the left ToC bar */ +img.sidebar-logo { + /* furo sets this at 100% but that makes it a bit too big */ + max-width: 85%; +} + +/* for sphinxcontrib.video */ +video { + width: 700px; + max-width: 100%; +} + +/* Do not underline links in the search results */ +#search-results a { + text-decoration: none; +} + +/* styling for added the source link component in the left ToC bar */ +.gh-source { + display: flex; + align-items: center; + gap: .5em; + padding-left: var(--sidebar-item-spacing-horizontal); + padding-right: var(--sidebar-item-spacing-horizontal); + padding-top: .6em; + padding-bottom: .6em; + text-decoration: none; + border-top: 1px solid var(--color-background-border); + border-bottom: 1px solid var(--color-background-border); +} + +.gh-source--icon { + height: 1.5em; +} + +.gh-source:hover .gh-source--info * { + color: var(--color-foreground-primary); +} + +.gh-source--info { + display: inline-flex; + flex-direction: column; + gap: .1em; +} + +.gh-source--version { + display: inline-flex; + align-items: center; + gap: .2em; +} + +.gh-source--version-icon { + height: .8em; +} + +.gh-source--version-icon, +.gh-source--version-text, +.gh-source--repo-text { + font-size: .8em; + color: var(--color-foreground-muted); +} + + +/** styling for the flowchart diagram on the landing page **/ +svg.sn-flow-chart path.text { + fill: var(--color-foreground-primary); +} + +svg.sn-flow-chart rect.box { + stroke: var(--color-foreground-border); +} + +svg.sn-flow-chart path.arrow { + fill: var(--ub-color-brand-main); +} + +/* Image width fix in need-sidebars. */ +tbody div.needs_side img.needs_image { + max-width: 100px; +} + +/** sphinx-design additional styling **/ +svg.fill-primary { + fill: var(--sd-color-primary); +} + +details.sd-dropdown { + margin: 1rem auto; +} + +summary.sd-summary-title { + padding-right: .5em !important; + /* note this can be removed in sphinx-design v0.6.1 */ +} + +.sn-dropdown-default .sd-summary-icon svg { + color: var(--color-admonition-title--note); +} + +.sn-dropdown-default { + border-left: .2rem solid var(--color-admonition-title--note) !important; +} + +.sn-dropdown-default .sd-summary-title { + border-width: 0 !important; +} diff --git a/sphinx-codelinks-logo_dark.svg b/docs/source/_static/sphinx-codelinks-logo_dark.svg similarity index 100% rename from sphinx-codelinks-logo_dark.svg rename to docs/source/_static/sphinx-codelinks-logo_dark.svg diff --git a/sphinx-codelinks-logo_light.svg b/docs/source/_static/sphinx-codelinks-logo_light.svg similarity index 100% rename from sphinx-codelinks-logo_light.svg rename to docs/source/_static/sphinx-codelinks-logo_light.svg diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..3aebce7 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,61 @@ +CodeLinks +========= + +``CodeLinks`` is a sphinx extension that provides a directive ``src-trace`` to trace the needs of source files. + +Usage +----- + +.. code-block:: rst + + .. src-trace:: example_with_file + :project: project_config + :file: example.cpp + + +or + +.. code-block:: rst + + .. src-trace:: example_with_directory + :project: project_config + :directory: ./example + + +``src-trace`` directive has the following options: + +* **project**: the project config specified in ``conf.py`` or ``toml`` to be used for source tracing. +* **file**: the source file to be traced. +* **directory**: the source files in the directory to be traced recursively. + +Regarding the options **file** and **directory**: + +- they are optional and mutually exclusive. +- the given paths of them are relative to ``src_dir`` defined in the source tracing configuration +- if not given, the whole project will be examined. + +**Example** + +.. src-trace:: dcdc demo_1 + :project: dcdc + :file: ./charge/demo_1.cpp + +.. src-trace:: dcdc charge + :project: dcdc + :directory: ./discharge + +Config +------ + +The config for source tracing can be specified in ``conf.py`` or ``toml`` file. +In the case where the config is introduced in ``toml`` file, the config path needs to be specified in ``conf.py`` + +.. code-block:: python + + # Specify the config path for source tracing in conf.py + src_trace_config_from_toml = "src_trace.toml" + +**Example Config** + + .. literalinclude:: ./../src_trace.toml + :language: toml diff --git a/docs/src_trace.toml b/docs/src_trace.toml new file mode 100644 index 0000000..a3f69cb --- /dev/null +++ b/docs/src_trace.toml @@ -0,0 +1,29 @@ +[src_trace] +# Configuration for source tracing +set_local_url = true # Set to true to enable local code html and URL generation +local_url_field = "local-url" # Need's field name for local URL +set_remote_url = true # Set to true to enable remote url to be generated +remote_url_field = "remote-url" # Need's field name for remote URL + +[src_trace.projects.dcdc] +# Configuration for source tracing project "dcdc" +comment_type = "cpp" # Type of the comment, only support C/C++ for now +src_dir = "../tests/data/dcdc" # Relative path from conf.py to the source directory +remote_url_pattern = "https://github.com/useblocks/sphinx_codelinks/blob/{commit}/{path}#L{line}" # URL pattern for remote source code +exclude = ["dcdc/src/ubt/ubt.cpp"] # Exclude files from source tracing +include = ["**/*.cpp", "**/*.hpp"] # Include files for source tracing +gitignore = true # Respect .gitignore to filter files + +[src_trace.projects.dcdc.oneline_comment_style] +# Configuration for oneline comment style +start_sequence = "[[" # Start sequence for oneline comments +end_sequence = "]]" # End sequence for the online comments; default is newline character +field_split_char = "," # Character to split fields in the comment +# Fields that are defined in the oneline comment style +needs_fields = [ + { "name" = "id", "type" = "str" }, + { "name" = "title", "type" = "str" }, + { "name" = "type", "type" = "str", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, +] diff --git a/docs/ubproject.toml b/docs/ubproject.toml new file mode 100644 index 0000000..5429bcf --- /dev/null +++ b/docs/ubproject.toml @@ -0,0 +1,12 @@ +"$schema" = "https://ubcode.useblocks.com/ubproject.schema.json" + +[rst_lint] +ignore = ["block.title_line"] + +[needs] +id_required = true + +[[needs.types]] +directive = "my-req" +title = "My Requirement" +prefix = "M_" From 1e600e4207ec9fe527705616874537b7609bd20b Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 12 Jun 2025 10:03:10 +0200 Subject: [PATCH 07/54] remove leading spaces --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 3aebce7..a16aeb1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -57,5 +57,5 @@ In the case where the config is introduced in ``toml`` file, the config path nee **Example Config** - .. literalinclude:: ./../src_trace.toml +.. literalinclude:: ./../src_trace.toml :language: toml From 177adfd59b3d918977945dc42e01ca97530687f9 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 12 Jun 2025 11:39:22 +0200 Subject: [PATCH 08/54] updated remote-url --- docs/src_trace.toml | 2 +- tests/data/sphinx/conf.py | 2 +- tests/data/sphinx/src_trace.toml | 2 +- tests/doc_test/recursive_dirs/src_trace.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src_trace.toml b/docs/src_trace.toml index a3f69cb..1f68587 100644 --- a/docs/src_trace.toml +++ b/docs/src_trace.toml @@ -9,7 +9,7 @@ remote_url_field = "remote-url" # Need's field name for remote URL # Configuration for source tracing project "dcdc" comment_type = "cpp" # Type of the comment, only support C/C++ for now src_dir = "../tests/data/dcdc" # Relative path from conf.py to the source directory -remote_url_pattern = "https://github.com/useblocks/sphinx_codelinks/blob/{commit}/{path}#L{line}" # URL pattern for remote source code +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" # URL pattern for remote source code exclude = ["dcdc/src/ubt/ubt.cpp"] # Exclude files from source tracing include = ["**/*.cpp", "**/*.hpp"] # Include files for source tracing gitignore = true # Respect .gitignore to filter files diff --git a/tests/data/sphinx/conf.py b/tests/data/sphinx/conf.py index 8b2c00e..8842a1c 100644 --- a/tests/data/sphinx/conf.py +++ b/tests/data/sphinx/conf.py @@ -78,7 +78,7 @@ # "dcdc": { # "type": "cpp", # "src_dir": "../../dcdc", # relative to confdir -# "remote_url_pattern": "https://github.com/useblocks/ubtrace/blob/{commit}/{path}#L{line}", # optional +# "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}", # optional # "exclude": ["dcdc/src/ubt/ubt.cpp"], # "include": ["**/*.cpp", "**/*.hpp"], # has default for each type # "gitignore": True, # default is True diff --git a/tests/data/sphinx/src_trace.toml b/tests/data/sphinx/src_trace.toml index b117ce5..cba8aa2 100644 --- a/tests/data/sphinx/src_trace.toml +++ b/tests/data/sphinx/src_trace.toml @@ -8,7 +8,7 @@ debug_measurement = true [src_trace.projects.dcdc] comment_type = "cpp" src_dir = "../dcdc" -remote_url_pattern = "https://github.com/useblocks/ubtrace/blob/{commit}/{path}#L{line}" +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" exclude = ["dcdc/src/ubt/ubt.cpp"] include = ["**/*.cpp", "**/*.hpp"] gitignore = true diff --git a/tests/doc_test/recursive_dirs/src_trace.toml b/tests/doc_test/recursive_dirs/src_trace.toml index da1d164..077df29 100644 --- a/tests/doc_test/recursive_dirs/src_trace.toml +++ b/tests/doc_test/recursive_dirs/src_trace.toml @@ -8,7 +8,7 @@ debug_measurement = true [src_trace.projects.dummy_src] comment_type = "cpp" src_dir = "./dummy_src_lv1" -remote_url_pattern = "https://github.com/useblocks/ubtrace/blob/{commit}/{path}#L{line}" +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" exclude = ["dcdc/src/ubt/ubt.cpp"] include = ["**/*.cpp", "**/*.hpp"] gitignore = true From c822c4fdeeddb8b15166822aeef5c26a12a56032 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 12 Jun 2025 11:42:50 +0200 Subject: [PATCH 09/54] adapted new repo --- src/sphinx_codelinks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sphinx_codelinks/README.md b/src/sphinx_codelinks/README.md index db188d8..de5c307 100644 --- a/src/sphinx_codelinks/README.md +++ b/src/sphinx_codelinks/README.md @@ -17,7 +17,7 @@ The project consists of the following three components: - Source Tracing: Sphinx extension to represent the collected the needs in the documentation `Source Discovery` and `Virtual Docs` can be used as `APIs` or `CLI tools`. -The detail usages can be found in the [test cases](./python/sphinx_codelinks/tests). +The detail usages can be found in the [test cases](./tests). The library is built to be From d55a864b82d18c7e9cae38c19e7c685995f7948f Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Mon, 16 Jun 2025 17:35:57 +0200 Subject: [PATCH 10/54] fix bugs and improve config validation --- src/sphinx_codelinks/cmd.py | 4 +- .../source_discovery/config.py | 68 ++++++++ .../{ => source_discovery}/source_discover.py | 0 .../sphinx_extension/config.py | 135 ++++++++++++++- .../sphinx_extension/directives/src_trace.py | 2 +- .../sphinx_extension/source_tracing.py | 17 +- src/sphinx_codelinks/virtual_docs/config.py | 73 +++++++-- tests/test_source_discover.py | 59 ++++++- tests/test_src_trace.py | 155 ++++++++++++++++++ tests/test_virtual_docs.py | 47 +++++- 10 files changed, 533 insertions(+), 27 deletions(-) create mode 100644 src/sphinx_codelinks/source_discovery/config.py rename src/sphinx_codelinks/{ => source_discovery}/source_discover.py (100%) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 8e5a2f5..80c5e0b 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -52,7 +52,7 @@ def discover( ] = None, ) -> None: """Discover the filepaths from the given root directory.""" - from sphinx_codelinks.source_discover import SourceDiscover + from sphinx_codelinks.source_discovery.source_discover import SourceDiscover source_discover = SourceDiscover( root_dir=root_dir, @@ -116,7 +116,7 @@ def vdoc( raise typer.BadParameter( f"Invalid oneline comment style configuration: {linesep.join(errors)}" ) - from sphinx_codelinks.source_discover import SourceDiscover + from sphinx_codelinks.source_discovery.source_discover import SourceDiscover from sphinx_codelinks.virtual_docs.utils import get_file_types file_types = get_file_types(comment_type) diff --git a/src/sphinx_codelinks/source_discovery/config.py b/src/sphinx_codelinks/source_discovery/config.py new file mode 100644 index 0000000..fc8212c --- /dev/null +++ b/src/sphinx_codelinks/source_discovery/config.py @@ -0,0 +1,68 @@ +from dataclasses import MISSING, dataclass, field, fields +from pathlib import Path +from typing import Any, TypedDict, cast + +from jsonschema import ValidationError, validate + + +class SourceDiscoveryConfigType(TypedDict): + root_dir: Path + excludes: list[str] + includes: list[str] + gitignore: bool + file_types: list[str] + + +@dataclass +class SourceDiscoveryConfig: + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + root_dir: Path = field( + default_factory=lambda: Path.cwd(), metadata={"schema": {"type": "string"}} + ) + """The root of the source directory.""" + + exclude: list[str] = field( + default_factory=list, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """The glob pattern to exclude files.""" + + include: list[str] = field( + default_factory=list, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """The glob pattern to include files.""" + + gitignore: bool = field(default=True, metadata={"schema": {"type": "boolean"}}) + """Whether to respect .gitignore to exclude files.""" + + file_types: list[str] = field( + default_factory=lambda: ["c", "h", "cpp", "hpp"], + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """The file types to discover.""" + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[explicit-any] + return None + + def check_schema(self) -> list[str]: + errors = [] + for _field_name in self.field_names(): + schema = self.get_schema(_field_name) + value = getattr(self, _field_name) + if isinstance(value, Path): # adapt to json schema restriction + value = str(value) + try: + validate(instance=value, schema=schema) # type: ignore[arg-type] # validate has no type specified + except ValidationError as e: + errors.append( + f"Schema validation error in field '{_field_name}': {e.message}" + ) + return errors diff --git a/src/sphinx_codelinks/source_discover.py b/src/sphinx_codelinks/source_discovery/source_discover.py similarity index 100% rename from src/sphinx_codelinks/source_discover.py rename to src/sphinx_codelinks/source_discovery/source_discover.py diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index 4ebe16b..06b0b68 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -1,10 +1,14 @@ from dataclasses import MISSING, dataclass, field, fields +from pathlib import Path from typing import Any, Literal, TypedDict, cast +from jsonschema import ValidationError, validate from sphinx.application import Sphinx from sphinx.config import Config as _SphinxConfig +from sphinx_codelinks.source_discovery.config import SourceDiscoveryConfig from sphinx_codelinks.virtual_docs.config import ( + SUPPORTED_COMMENT_TYPES, OneLineCommentStyle, OneLineCommentStyleType, ) @@ -88,6 +92,14 @@ def add_config_values(cls, app: Sphinx) -> None: def field_names(cls) -> set[str]: return {item.name for item in fields(cls)} + @classmethod + def get_schema(cls, name: str) -> dict | None: + """Get the schema for a config item.""" + _field = next(field for field in fields(cls) if field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return _field.metadata["schema"] + return None + config_from_toml: str | None = field( default=None, metadata={ @@ -106,13 +118,22 @@ def field_names(cls) -> set[str]: metadata={ "rebuild": "env", "types": (bool,), + "schema": { + "type": "boolean", + }, }, ) """Set the file URL in the extracted need.""" local_url_field: str = field( default="local-url", - metadata={"rebuild": "env", "types": (str,)}, + metadata={ + "rebuild": "env", + "types": (str,), + "schema": { + "type": "string", + }, + }, ) """The field name for the file URL in the extracted need.""" @@ -121,17 +142,45 @@ def field_names(cls) -> set[str]: metadata={ "rebuild": "env", "types": (bool,), + "schema": { + "type": "boolean", + }, }, ) remote_url_field: str = field( default="remote-url", - metadata={"rebuild": "env", "types": (str,)}, + metadata={ + "rebuild": "env", + "types": (str,), + "schema": { + "type": "string", + }, + }, ) """The field name for the remote URL in the extracted need.""" projects: dict[str, SrcTraceProjectConfigType] = field( default_factory=dict, - metadata={"rebuild": "env", "types": ()}, + metadata={ + "rebuild": "env", + "types": (), + "schema": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "comment_type": {}, + "src_dir": {}, + "remote_url_pattern": {}, + "exclude": {}, + "include": {}, + "gitignore": {}, + "oneline_comment_style": {}, + }, + "additionalProperties": False, + }, + }, + }, ) """The configuration for the source tracing projects.""" @@ -143,3 +192,83 @@ def field_names(cls) -> set[str]: default=False, metadata={"rebuild": "html", "types": (bool,)} ) """If True, log filter processing runtime information.""" + + +def check_schema(config: SrcTraceSphinxConfig) -> list[str]: + errors = [] + for _field_name in SrcTraceSphinxConfig.field_names(): + schema = SrcTraceSphinxConfig.get_schema(_field_name) + value = getattr(config, _field_name) + if not schema: + continue + try: + validate(instance=value, schema=schema) + except ValidationError as e: + errors.append( + f"Schema validation error in filed '{_field_name}': {e.message}" + ) + return errors + + +def check_project_configuration(config: SrcTraceSphinxConfig) -> list[str]: + errors = [] + + def validate_oneline_comment_style(project_config): + if "oneline_comment_style" in project_config: + return project_config["oneline_comment_style"].check_fields_configuration() + return [] + + def build_src_discovery_dict(project_config): + src_discovery_dict: dict[str, Any] = {} + src_discovery_errors = [] + if "src_dir" in project_config: + if isinstance(project_config["src_dir"], str): + src_discovery_dict["root_dir"] = Path(project_config["src_dir"]) + else: + src_discovery_errors.append("src_dir must be a string") + for key in ("exclude", "include", "gitignore"): + if key in project_config: + src_discovery_dict[key] = project_config[key] + if "comment_type" in project_config: + if project_config["comment_type"] not in SUPPORTED_COMMENT_TYPES: + src_discovery_errors.append( + f"comment_type must be one of {sorted(SUPPORTED_COMMENT_TYPES)}" + ) + else: + src_discovery_dict["file_types"] = list(SUPPORTED_COMMENT_TYPES) + return src_discovery_dict, src_discovery_errors + + for project_name, project_config in config.projects.items(): + project_errors = [] + oneline_errors = validate_oneline_comment_style(project_config) + src_discovery_dict, src_discovery_errors = build_src_discovery_dict( + project_config + ) + if src_discovery_dict is not None: + src_discovery_config = SourceDiscoveryConfig(**src_discovery_dict) + src_discovery_errors.extend(src_discovery_config.check_schema()) + + if config.set_remote_url and "remote_url_pattern" not in project_config: + project_errors.append( + "remote_url_pattern must be given, as set_remote_url is enabled" + ) + + if "remote_url_pattern" in project_config and not isinstance( + project_config["remote_url_pattern"], str + ): + project_errors.append("remote_url_pattern must be a string") + + if oneline_errors or src_discovery_errors or project_errors: + errors.append(f"Project '{project_name}' has the following errors:") + errors.extend(oneline_errors) + errors.extend(src_discovery_errors) + errors.extend(project_errors) + + return errors + + +def check_configuration(config: SrcTraceSphinxConfig) -> list[str]: + errors = [] + errors.extend(check_schema(config)) + errors.extend(check_project_configuration(config)) + return errors diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py index 20c72d1..25da818 100644 --- a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -11,7 +11,7 @@ from sphinx_needs.api import add_need # type: ignore[import-untyped] from sphinx_needs.utils import add_doc # type: ignore[import-untyped] -from sphinx_codelinks.source_discover import SourceDiscover +from sphinx_codelinks.source_discovery.source_discover import SourceDiscover from sphinx_codelinks.sphinx_extension.config import ( SRC_TRACE_CACHE, SrcTraceProjectConfigType, diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py index edecca3..c1b6a32 100644 --- a/src/sphinx_codelinks/sphinx_extension/source_tracing.py +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -6,7 +6,7 @@ from typing import Any from sphinx.application import Sphinx -from sphinx.config import Config +from sphinx.config import Config as _SphinxConfig from sphinx.environment import BuildEnvironment from sphinx.util import logging from sphinx.util.fileutil import copy_asset @@ -107,7 +107,7 @@ def generate_code_page( return None -def load_config_from_toml(app: Sphinx, config: Config) -> None: +def load_config_from_toml(app: Sphinx, config: _SphinxConfig) -> None: """Load the configuration from a TOML file, if defined in conf.py.""" src_trc_sphinx_config = SrcTraceSphinxConfig(config) if src_trc_sphinx_config.config_from_toml is None: @@ -136,8 +136,14 @@ def load_config_from_toml(app: Sphinx, config: Config) -> None: ) return + set_config_to_sphinx(toml_data, config) + + +def set_config_to_sphinx( + src_trace_config: dict[str, Any], config: _SphinxConfig +) -> None: allowed_keys = SrcTraceSphinxConfig.field_names() - for key, value in toml_data.items(): + for key, value in src_trace_config.items(): if key not in allowed_keys: continue if key == "projects": @@ -149,10 +155,11 @@ def load_config_from_toml(app: Sphinx, config: Config) -> None: project_config["oneline_comment_style"] = OneLineCommentStyle( **project_config["oneline_comment_style"] ) + config[f"src_trace_{key}"] = value -def update_sn_extra_options(app: Sphinx, config: Config) -> None: +def update_sn_extra_options(app: Sphinx, config: _SphinxConfig) -> None: src_trace_sphinx_config = SrcTraceSphinxConfig(config) add_extra_option(app, "project") add_extra_option(app, "file") @@ -163,7 +170,7 @@ def update_sn_extra_options(app: Sphinx, config: Config) -> None: add_extra_option(app, src_trace_sphinx_config.remote_url_field) -def update_sn_types(app: Sphinx, _config: Config) -> None: +def update_sn_types(app: Sphinx, _config: _SphinxConfig) -> None: add_need_type(app, "srctrace", "Src-Trace", "ST_", "#ffffff", "node") diff --git a/src/sphinx_codelinks/virtual_docs/config.py b/src/sphinx_codelinks/virtual_docs/config.py index f330f16..94c1e12 100644 --- a/src/sphinx_codelinks/virtual_docs/config.py +++ b/src/sphinx_codelinks/virtual_docs/config.py @@ -1,6 +1,7 @@ from dataclasses import MISSING, dataclass, field, fields import logging import os +from pathlib import Path from typing import Any, Literal, TypedDict, cast from jsonschema import ValidationError, validate @@ -17,10 +18,65 @@ SUPPORTED_COMMENT_TYPES = {"c", "h", "cpp", "hpp"} +class VirtualDocsConfigType(TypedDict): + src_files: list[Path] | None + src_dir: Path + output_dir: Path + comment_type: str + + +@dataclass +class VirtualDocsConfig: + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + src_files: list[Path] = field( + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """A list of source files to be processed.""" + + src_dir: Path = field( + default_factory=lambda: Path.cwd(), metadata={"schema": {"type": "string"}} + ) + """The root of the source directory.""" + + output_dir: Path = field( + default=Path("output"), metadata={"schema": {"type": "string"}} + ) + """The directory where the virtual documents and their caches will be stored.""" + + comment_type: str = field(default="c", metadata={"schema": {"type": "string"}}) + """The type of comment to be processed.""" + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[explicit-any] + return None + + def check_schema(self) -> list[str]: + errors = [] + for _field_name in self.field_names(): + schema = self.get_schema(_field_name) + value = getattr(self, _field_name) + if _field_name == "src_files": # adapt to json schema restriction + if isinstance(value, list): + value: list[str] = [str(src_file) for src_file in value] + elif isinstance(value, Path): # adapt to json schema restriction + value = str(value) + try: + validate(instance=value, schema=schema) # type: ignore[arg-type] # validate has no type specified + except ValidationError as e: + errors.append( + f"Schema validation error in field '{_field_name}': {e.message}" + ) + return errors + + class FieldConfig(TypedDict, total=False): name: str - quoted: bool - named: bool type: Literal["str", "list[str]"] default: str | list[str] | None @@ -66,8 +122,6 @@ def field_names(cls) -> set[str]: "required_fields": ["title", "type"], "field_default": { "type": "str", - "quoted": False, - "named": False, }, "schema": { "type": "array", @@ -75,8 +129,6 @@ def field_names(cls) -> set[str]: "type": "object", "properties": { "name": {"type": "string"}, - "quoted": {"type": "boolean", "default": False}, - "named": {"type": "boolean", "default": False}, "type": { "type": "string", "enum": ["str", "list[str]"], @@ -165,11 +217,10 @@ def check_required_fields(self) -> list[str]: if required_fields is None: errors.append("No required fields specified.") return errors - for _field in self.needs_fields: - if _field["name"] in required_fields: - required_fields.remove(_field["name"]) - if len(required_fields) != 0: - errors.append(f"Missing required fields: {required_fields}") + given_field_names = [_field["name"] for _field in self.needs_fields] + missing_fields = set(required_fields) - set(given_field_names) + if len(missing_fields) != 0: + errors.append(f"Missing required fields: {sorted(missing_fields)}") return errors diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index 3e99612..d74aa26 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -1,6 +1,63 @@ from pathlib import Path -from sphinx_codelinks.source_discover import SourceDiscover +import pytest + +from sphinx_codelinks.source_discovery.config import SourceDiscoveryConfig +from sphinx_codelinks.source_discovery.source_discover import SourceDiscover + + +@pytest.mark.parametrize( + ("config", "msgs"), + [ + ( + { + "root_dir": 123, + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "file_types": ["cpp", "hpp"], + }, + [ + "Schema validation error in field 'root_dir': 123 is not of type 'string'" + ], + ), + ( + { + "root_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": "TrueAsString", + "file_types": ["cpp", "hpp"], + }, + [ + "Schema validation error in field 'gitignore': 'TrueAsString' is not of type 'boolean'" + ], + ), + ], +) +def test_schema_negative(config, msgs): + source_discovery_config = SourceDiscoveryConfig(**config) + errors = source_discovery_config.check_schema() + assert errors.sort() == msgs.sort() + + +@pytest.mark.parametrize( + "config", + [ + {}, + { + "root_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "file_types": ["cpp", "hpp"], + }, + ], +) +def test_schema_positive(config): + source_discovery_config = SourceDiscoveryConfig(**config) + errors = source_discovery_config.check_schema() + assert len(errors) == 0 def test_source_discover_all_files(source_directory: Path): diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py index 11053f0..41405a6 100644 --- a/tests/test_src_trace.py +++ b/tests/test_src_trace.py @@ -5,6 +5,161 @@ import pytest from sphinx.testing.util import SphinxTestApp +from sphinx_codelinks.sphinx_extension.config import ( + SrcTraceSphinxConfig, + check_configuration, +) +from sphinx_codelinks.sphinx_extension.source_tracing import set_config_to_sphinx + + +@pytest.mark.parametrize( + ("src_trace_config", "result"), + [ + ( + { + "remote_url_field": 555, + "local_url_field": 789, + "set_local_url": "fdd", + "set_remote_url": "TrueString", + "projects": { + "dcdc": { + "comment_type": "java", + "src_dir": ["../dcdc"], + "remote_url_pattern": 44332, + "exclude": [123], + "include": [345], + "gitignore": "_true", + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "list[]", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, + } + }, + }, + [ + "Schema validation error in filed 'remote_url_field': 555 is not of type 'string'", + "Schema validation error in filed 'set_remote_url': 'TrueString' is not of type 'boolean'", + "Schema validation error in filed 'local_url_field': 789 is not of type 'string'", + "Schema validation error in filed 'set_local_url': 'fdd' is not of type 'boolean'", + "Project 'dcdc' has the following errors:", + "Schema validation error in need_fields 'title': 'list[]' is not one of ['str', 'list[str]']", + "src_dir must be a string", + "comment_type must be one of ['c', 'cpp', 'h', 'hpp']", + "Schema validation error in field 'exclude': 123 is not of type 'string'", + "Schema validation error in field 'include': 345 is not of type 'string'", + "Schema validation error in field 'gitignore': '_true' is not of type 'boolean'", + "remote_url_pattern must be a string", + ], + ), + ( + { + "remote_url_field": "remote-url", + "local_url_field": "local-url", + "set_local_url": True, + "set_remote_url": True, + "projects": { + "dcdc": { + "comment_type": "cpp", + "src_dir": "../dcdc", + # intentionally not given "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}", + "exclude": [], + "include": [], + "gitignore": True, + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "str", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, + } + }, + }, + [ + "Project 'dcdc' has the following errors:", + "remote_url_pattern must be given, as set_remote_url is enabled", + ], + ), + ], +) +def test_src_tracing_config_negative( + make_app: Callable[..., SphinxTestApp], + src_trace_config, + result, +): + this_file_dir = Path(__file__).parent + sphinx_project = Path("data") / "sphinx" + app = make_app(srcdir=(this_file_dir / sphinx_project)) + set_config_to_sphinx(src_trace_config, app.env.config) + src_trace_sphinx_config = SrcTraceSphinxConfig(app.env.config) + errors = check_configuration(src_trace_sphinx_config) + assert sorted(errors) == sorted(result) + + +def test_src_tracing_config_positive( + make_app: Callable[..., SphinxTestApp], +): + src_trace_config = { + "remote_url_field": "remote-url", + "local_url_field": "local-url", + "set_local_url": True, + "set_remote_url": True, + "projects": { + "dcdc": { + "comment_type": "cpp", + "src_dir": "../dcdc", + "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}", + "exclude": ["**/*.hpp"], + "include": ["**/*.cpp"], + "gitignore": True, + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "str", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, + } + }, + } + this_file_dir = Path(__file__).parent + sphinx_project = Path("data") / "sphinx" + app = make_app(srcdir=(this_file_dir / sphinx_project)) + set_config_to_sphinx(src_trace_config, app.env.config) + src_trace_sphinx_config = SrcTraceSphinxConfig(app.env.config) + errors = check_configuration(src_trace_sphinx_config) + assert not errors + @pytest.mark.parametrize( ("sphinx_project", "source_code"), diff --git a/tests/test_virtual_docs.py b/tests/test_virtual_docs.py index 591e89d..42dc8bf 100644 --- a/tests/test_virtual_docs.py +++ b/tests/test_virtual_docs.py @@ -2,7 +2,11 @@ import pytest -from sphinx_codelinks.virtual_docs.config import ESCAPE, OneLineCommentStyle +from sphinx_codelinks.virtual_docs.config import ( + ESCAPE, + OneLineCommentStyle, + VirtualDocsConfig, +) from sphinx_codelinks.virtual_docs.utils import ( OnelineParserInvalidWarning, WarningSubTypeEnum, @@ -29,6 +33,41 @@ ONELINE_COMMENT_STYLE_DEFAULT = OneLineCommentStyle() +@pytest.mark.parametrize( + ("vdocs_config", "result"), + [ + ( + VirtualDocsConfig( + src_files=[ + TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", + ], + src_dir=TEST_DIR / "data" / "dcdc", + output_dir=TEST_DIR / "output", + comment_type=123, + ), + [ + "Schema validation error in field 'comment_type': 123 is not of type 'string'", + ], + ), + ( + VirtualDocsConfig( + src_files=None, + src_dir=TEST_DIR / "data" / "dcdc", + output_dir=TEST_DIR / "output", + comment_type=123, + ), + [ + "Schema validation error in field 'comment_type': 123 is not of type 'string'", + "Schema validation error in field 'src_files': None is not of type 'array'", + ], + ), + ], +) +def test_config_schema_validator_negative(vdocs_config, result): + errors = vdocs_config.check_schema() + assert sorted(errors) == sorted(result) + + @pytest.mark.parametrize( "oneline_config, result", [ @@ -144,9 +183,9 @@ ), ], ) -def test_schema_validator_negative(oneline_config, result): +def test_oneline_schema_validator_negative(oneline_config, result): errors = oneline_config.check_fields_configuration() - assert errors.sort() == result.sort() + assert sorted(errors) == sorted(result) @pytest.mark.parametrize( @@ -180,7 +219,7 @@ def test_schema_validator_negative(oneline_config, result): ), ], ) -def test_schema_validator_positive(oneline_config): +def test_oneline_schema_validator_positive(oneline_config): assert len(oneline_config.check_fields_configuration()) == 0 From d47566fb15b6fa1562995ea221d2ef69e414bdb1 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Mon, 16 Jun 2025 17:43:39 +0200 Subject: [PATCH 11/54] added configuration for src-trace --- src/sphinx_codelinks/sphinx_extension/source_tracing.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py index c1b6a32..bd69884 100644 --- a/src/sphinx_codelinks/sphinx_extension/source_tracing.py +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -19,6 +19,7 @@ from sphinx_codelinks.sphinx_extension.config import ( SRC_TRACE_CACHE, SrcTraceSphinxConfig, + check_configuration, file_lineno_href, ) from sphinx_codelinks.sphinx_extension.directives.src_trace import ( @@ -45,6 +46,7 @@ def setup(app: Sphinx) -> dict[str, Any]: # type: ignore[explicit-any] "config-inited", update_sn_extra_options, priority=11 ) # run early otherwise, extra options are not set for nested_parse app.connect("config-inited", update_sn_types) + app.connect("config-inited", check_sphinx_configuration) app.connect("env-before-read-docs", prepare_env) app.connect("html-collect-pages", generate_code_page) @@ -190,6 +192,13 @@ def prepare_env(app: Sphinx, env: BuildEnvironment, _docnames: list[str]) -> Non Path(str(app.outdir), "debug_filters.jsonl").unlink() +def check_sphinx_configuration(app: Sphinx, _config: _SphinxConfig) -> None: + config = SrcTraceSphinxConfig(app.config) + errors = check_configuration(config) + if errors: + raise Exception("\n".join(errors)) + + def emit_warnings( app: Sphinx, _env: BuildEnvironment, From c2c2ba614def99198df28949eeec5dca73855646 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Mon, 16 Jun 2025 21:10:01 +0200 Subject: [PATCH 12/54] fixed mypy errors --- .../source_discovery/config.py | 6 ++-- .../sphinx_extension/config.py | 36 ++++++++++++++----- .../sphinx_extension/source_tracing.py | 26 +++++++++----- src/sphinx_codelinks/virtual_docs/config.py | 3 +- 4 files changed, 50 insertions(+), 21 deletions(-) diff --git a/src/sphinx_codelinks/source_discovery/config.py b/src/sphinx_codelinks/source_discovery/config.py index fc8212c..e6eb62b 100644 --- a/src/sphinx_codelinks/source_discovery/config.py +++ b/src/sphinx_codelinks/source_discovery/config.py @@ -5,10 +5,10 @@ from jsonschema import ValidationError, validate -class SourceDiscoveryConfigType(TypedDict): +class SourceDiscoveryConfigType(TypedDict, total=False): root_dir: Path - excludes: list[str] - includes: list[str] + exclude: list[str] + include: list[str] gitignore: bool file_types: list[str] diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index 06b0b68..03851e7 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -6,7 +6,10 @@ from sphinx.application import Sphinx from sphinx.config import Config as _SphinxConfig -from sphinx_codelinks.source_discovery.config import SourceDiscoveryConfig +from sphinx_codelinks.source_discovery.config import ( + SourceDiscoveryConfig, + SourceDiscoveryConfigType, +) from sphinx_codelinks.virtual_docs.config import ( SUPPORTED_COMMENT_TYPES, OneLineCommentStyle, @@ -37,6 +40,17 @@ class SrcTraceProjectConfigType(TypedDict): oneline_comment_style: OneLineCommentStyle +class SrcTraceConfigType(TypedDict): + config_from_toml: str | None + set_local_url: bool + local_url_field: str + set_remote_url: bool + remote_url_field: str + projects: dict[str, SrcTraceProjectConfigType] + debug_measurement: bool + debug_filters: bool + + @dataclass class SrcTraceSphinxConfig: def __init__(self, config: _SphinxConfig) -> None: @@ -93,11 +107,11 @@ def field_names(cls) -> set[str]: return {item.name for item in fields(cls)} @classmethod - def get_schema(cls, name: str) -> dict | None: + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] """Get the schema for a config item.""" _field = next(field for field in fields(cls) if field.name is name) if _field.metadata is not MISSING and "schema" in _field.metadata: - return _field.metadata["schema"] + return _field.metadata["schema"] # type: ignore[no-any-return] return None config_from_toml: str | None = field( @@ -213,13 +227,19 @@ def check_schema(config: SrcTraceSphinxConfig) -> list[str]: def check_project_configuration(config: SrcTraceSphinxConfig) -> list[str]: errors = [] - def validate_oneline_comment_style(project_config): + def validate_oneline_comment_style( + project_config: SrcTraceProjectConfigType, + ) -> list[str]: if "oneline_comment_style" in project_config: - return project_config["oneline_comment_style"].check_fields_configuration() + style = project_config["oneline_comment_style"] + if isinstance(style, OneLineCommentStyle): + return style.check_fields_configuration() return [] - def build_src_discovery_dict(project_config): - src_discovery_dict: dict[str, Any] = {} + def build_src_discovery_dict( + project_config: SrcTraceProjectConfigType, + ) -> tuple[SourceDiscoveryConfigType | None, list[str]]: + src_discovery_dict = cast(SourceDiscoveryConfigType, {}) src_discovery_errors = [] if "src_dir" in project_config: if isinstance(project_config["src_dir"], str): @@ -239,7 +259,7 @@ def build_src_discovery_dict(project_config): return src_discovery_dict, src_discovery_errors for project_name, project_config in config.projects.items(): - project_errors = [] + project_errors: list[str] = [] oneline_errors = validate_oneline_comment_style(project_config) src_discovery_dict, src_discovery_errors = build_src_discovery_dict( project_config diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py index bd69884..21b7cf8 100644 --- a/src/sphinx_codelinks/sphinx_extension/source_tracing.py +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -3,7 +3,7 @@ from pathlib import Path from timeit import default_timer as timer # Used for timing measurements import tomllib -from typing import Any +from typing import Any, cast from sphinx.application import Sphinx from sphinx.config import Config as _SphinxConfig @@ -18,6 +18,8 @@ from sphinx_codelinks.sphinx_extension import debug from sphinx_codelinks.sphinx_extension.config import ( SRC_TRACE_CACHE, + SrcTraceConfigType, + SrcTraceProjectConfigType, SrcTraceSphinxConfig, check_configuration, file_lineno_href, @@ -121,8 +123,7 @@ def load_config_from_toml(app: Sphinx, config: _SphinxConfig) -> None: if not toml_file.exists(): logger.warning( - f"Source tracing configuration file {toml_file} does not exist. " - "Using configuration from conf.py." + f"Source tracing configuration file {toml_file} does not exist. Using configuration from conf.py." ) return try: @@ -138,24 +139,31 @@ def load_config_from_toml(app: Sphinx, config: _SphinxConfig) -> None: ) return - set_config_to_sphinx(toml_data, config) + set_config_to_sphinx( + src_trace_config=cast(SrcTraceConfigType, toml_data), config=config + ) def set_config_to_sphinx( - src_trace_config: dict[str, Any], config: _SphinxConfig + src_trace_config: SrcTraceConfigType, config: _SphinxConfig ) -> None: allowed_keys = SrcTraceSphinxConfig.field_names() for key, value in src_trace_config.items(): if key not in allowed_keys: continue if key == "projects": - for project_config in value.values(): - oneline_comment_style: OneLineCommentStyleType | None = ( - project_config.get("oneline_comment_style") + for project_config in cast( + dict[str, SrcTraceProjectConfigType], value + ).values(): + oneline_comment_style: OneLineCommentStyleType | None = cast( + OneLineCommentStyleType, project_config.get("oneline_comment_style") ) if oneline_comment_style: project_config["oneline_comment_style"] = OneLineCommentStyle( - **project_config["oneline_comment_style"] + **cast( + OneLineCommentStyleType, + project_config["oneline_comment_style"], + ) ) config[f"src_trace_{key}"] = value diff --git a/src/sphinx_codelinks/virtual_docs/config.py b/src/sphinx_codelinks/virtual_docs/config.py index 94c1e12..6a4fbfa 100644 --- a/src/sphinx_codelinks/virtual_docs/config.py +++ b/src/sphinx_codelinks/virtual_docs/config.py @@ -63,9 +63,10 @@ def check_schema(self) -> list[str]: value = getattr(self, _field_name) if _field_name == "src_files": # adapt to json schema restriction if isinstance(value, list): - value: list[str] = [str(src_file) for src_file in value] + value: list[str] = [str(src_file) for src_file in value] # type: ignore[no-redef] # only for value adaptation elif isinstance(value, Path): # adapt to json schema restriction value = str(value) + try: validate(instance=value, schema=schema) # type: ignore[arg-type] # validate has no type specified except ValidationError as e: From d897edb213e71a823f34a5e6aea32951606eaa71 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Tue, 24 Jun 2025 16:48:21 +0200 Subject: [PATCH 13/54] improve config validation --- pyproject.toml | 4 + src/sphinx_codelinks/cmd.py | 91 +++++++++++-------- .../source_discovery/source_discover.py | 12 +-- .../sphinx_extension/config.py | 64 ++++++------- .../sphinx_extension/directives/src_trace.py | 4 +- tests/test_cmd.py | 89 ++++++++++++++++++ tests/test_source_discover.py | 6 +- 7 files changed, 188 insertions(+), 82 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f122feb..26af91a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,8 +49,12 @@ dev-dependencies = [ "shiv>=1.0.8", "insta-science>=0.2.1", "types-jsonschema>=4.23.0.20241208", + "toml>=0.10.2", ] +[project.scripts] +cl = "sphinx_codelinks.cmd:app" + [tool.rye.scripts] # linting and formatting "mypy:all" = "mypy ." diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 80c5e0b..95735ac 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -1,11 +1,20 @@ +from collections import deque from os import linesep from pathlib import Path import tempfile import tomllib -from typing import Annotated, Any +from typing import Annotated, cast import typer +from sphinx_codelinks.source_discovery.config import SourceDiscoveryConfig +from sphinx_codelinks.sphinx_extension.config import ( + SrcTraceProjectConfigType, + build_src_discovery_dict, + validate_oneline_comment_style, +) +from sphinx_codelinks.virtual_docs.config import OneLineCommentStyle + app = typer.Typer( no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]} ) @@ -25,7 +34,7 @@ def discover( resolve_path=True, ), ], - excludes: Annotated[ + exclude: Annotated[ list[str] | None, typer.Option( "--excludes", @@ -33,7 +42,7 @@ def discover( help="Glob patterns to be excluded.", ), ] = None, - includes: Annotated[ + include: Annotated[ list[str] | None, typer.Option( "--includes", @@ -56,8 +65,8 @@ def discover( source_discover = SourceDiscover( root_dir=root_dir, - excludes=excludes, - includes=includes, + exclude=exclude, + include=include, file_types=file_types, gitignore=gitignore, ) @@ -92,51 +101,53 @@ def vdoc( ) -> None: """Generate virtual documents for caching and extract the oneline comments.""" - from sphinx_codelinks.virtual_docs.config import OneLineCommentStyle - data = load_config_from_toml(config, project) - # src_dir = Path(data["src_dir"]) - # # if not project: - # # src_tracing extension assume root_dir as the config - root_dir = config.parent - src_dir = (root_dir / Path(data["src_dir"])).resolve() - - comment_type = data["comment_type"] - gitignore = data["gitignore"] - excludes = data["exclude"] - includes = data["include"] - oneline_comment_style = data.get("oneline_comment_style") - if oneline_comment_style is None: - oneline_comment_style = OneLineCommentStyle() - else: - oneline_comment_style = OneLineCommentStyle(**oneline_comment_style) - errors = oneline_comment_style.check_fields_configuration() + errors: deque[str] = deque() + oneline_errors = validate_oneline_comment_style(data) + + if oneline_errors: + errors.appendleft("Invalid oneline comment style configuration:") + errors.extend(oneline_errors) + + src_discovery_dict, src_discovery_errors = build_src_discovery_dict(data) + src_discovery_config = SourceDiscoveryConfig(**src_discovery_dict) + src_discovery_errors.extend(src_discovery_config.check_schema()) + + if src_discovery_errors: + errors.appendleft("Invalid source discovery configuration:") + errors.extend(src_discovery_errors) + if errors: - raise typer.BadParameter( - f"Invalid oneline comment style configuration: {linesep.join(errors)}" - ) - from sphinx_codelinks.source_discovery.source_discover import SourceDiscover - from sphinx_codelinks.virtual_docs.utils import get_file_types + raise typer.BadParameter(f"{linesep.join(errors)}") - file_types = get_file_types(comment_type) + from sphinx_codelinks.source_discovery.source_discover import SourceDiscover + src_root_dir = (config.parent / src_discovery_config.root_dir).resolve() source_discover = SourceDiscover( - root_dir=src_dir, - excludes=excludes, - includes=includes, - file_types=file_types, - gitignore=gitignore, + root_dir=src_root_dir, + exclude=src_discovery_config.exclude, + include=src_discovery_config.include, + file_types=src_discovery_config.file_types, + gitignore=src_discovery_config.gitignore, ) from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs + oneline_comment_style = data.get("oneline_comment_style") + if oneline_comment_style is None: + oneline_comment_style = OneLineCommentStyle() + else: + oneline_comment_style = OneLineCommentStyle(**oneline_comment_style) + virtual_docs = VirtualDocs( src_files=source_discover.source_paths, - src_dir=str(src_dir), + src_dir=str(src_root_dir), output_dir=str(output_dir), oneline_comment_style=oneline_comment_style, - comment_type=comment_type, + comment_type=src_discovery_config.file_types[0] + if src_discovery_config.file_types + else "c", ) virtual_docs.collect() virtual_docs.dump_virtual_docs() @@ -145,7 +156,7 @@ def vdoc( typer.echo("The virtual documents are generated:") for v_doc in virtual_docs.virtual_docs: json_path = output_dir / v_doc.filepath.with_suffix(".json").relative_to( - src_dir + src_root_dir ) typer.echo(json_path) else: @@ -157,9 +168,9 @@ def vdoc( typer.echo(cached_file) -def load_config_from_toml( # type: ignore[explicit-any] +def load_config_from_toml( toml_file: Path, project: str | None = None -) -> dict[str, Any]: +) -> SrcTraceProjectConfigType: try: with toml_file.open("rb") as f: toml_data = tomllib.load(f) @@ -172,7 +183,7 @@ def load_config_from_toml( # type: ignore[explicit-any] f"Failed to load source tracing configuration from {toml_file}" ) from e - return toml_data + return cast(SrcTraceProjectConfigType, toml_data) if __name__ == "__main__": diff --git a/src/sphinx_codelinks/source_discovery/source_discover.py b/src/sphinx_codelinks/source_discovery/source_discover.py index badae11..f4853c4 100644 --- a/src/sphinx_codelinks/source_discovery/source_discover.py +++ b/src/sphinx_codelinks/source_discovery/source_discover.py @@ -10,14 +10,14 @@ class SourceDiscover: def __init__( self, root_dir: Path, - excludes: list[str] | None = None, - includes: list[str] | None = None, + exclude: list[str] | None = None, + include: list[str] | None = None, gitignore: bool = True, file_types: list[str] | None = None, ): self.root_path = root_dir - self.excludes = excludes - self.includes = includes + self.exclude = exclude + self.include = include # Only gitignore at source root is considered. # TODO: Support nested gitignore files gitignore_path = self.root_path / ".gitignore" @@ -48,7 +48,7 @@ def _discover(self) -> list[Path]: if self.file_types and filepath.suffix.lower() not in self.file_types: continue rel_filepath = str(filepath.relative_to(self.root_path)) - if self.includes and self._matches_any(rel_filepath, self.includes): + if self.include and self._matches_any(rel_filepath, self.include): # "includes" has the highest priority over "gitignore" and "excludes" discovered_files.append(filepath) continue @@ -56,7 +56,7 @@ def _discover(self) -> list[Path]: str(filepath.absolute()) ): continue - if self.excludes and self._matches_any(rel_filepath, self.excludes): + if self.exclude and self._matches_any(rel_filepath, self.exclude): continue discovered_files.append(filepath) sorted_filepaths = sorted( diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index 03851e7..3f1dcc7 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -227,37 +227,6 @@ def check_schema(config: SrcTraceSphinxConfig) -> list[str]: def check_project_configuration(config: SrcTraceSphinxConfig) -> list[str]: errors = [] - def validate_oneline_comment_style( - project_config: SrcTraceProjectConfigType, - ) -> list[str]: - if "oneline_comment_style" in project_config: - style = project_config["oneline_comment_style"] - if isinstance(style, OneLineCommentStyle): - return style.check_fields_configuration() - return [] - - def build_src_discovery_dict( - project_config: SrcTraceProjectConfigType, - ) -> tuple[SourceDiscoveryConfigType | None, list[str]]: - src_discovery_dict = cast(SourceDiscoveryConfigType, {}) - src_discovery_errors = [] - if "src_dir" in project_config: - if isinstance(project_config["src_dir"], str): - src_discovery_dict["root_dir"] = Path(project_config["src_dir"]) - else: - src_discovery_errors.append("src_dir must be a string") - for key in ("exclude", "include", "gitignore"): - if key in project_config: - src_discovery_dict[key] = project_config[key] - if "comment_type" in project_config: - if project_config["comment_type"] not in SUPPORTED_COMMENT_TYPES: - src_discovery_errors.append( - f"comment_type must be one of {sorted(SUPPORTED_COMMENT_TYPES)}" - ) - else: - src_discovery_dict["file_types"] = list(SUPPORTED_COMMENT_TYPES) - return src_discovery_dict, src_discovery_errors - for project_name, project_config in config.projects.items(): project_errors: list[str] = [] oneline_errors = validate_oneline_comment_style(project_config) @@ -292,3 +261,36 @@ def check_configuration(config: SrcTraceSphinxConfig) -> list[str]: errors.extend(check_schema(config)) errors.extend(check_project_configuration(config)) return errors + + +def validate_oneline_comment_style( + project_config: SrcTraceProjectConfigType, +) -> list[str]: + if "oneline_comment_style" in project_config: + style = project_config["oneline_comment_style"] + if isinstance(style, OneLineCommentStyle): + return style.check_fields_configuration() + return [] + + +def build_src_discovery_dict( + project_config: SrcTraceProjectConfigType, +) -> tuple[SourceDiscoveryConfigType | None, list[str]]: + src_discovery_dict = cast(SourceDiscoveryConfigType, {}) + src_discovery_errors = [] + if "src_dir" in project_config: + if isinstance(project_config["src_dir"], str): + src_discovery_dict["root_dir"] = Path(project_config["src_dir"]) + else: + src_discovery_errors.append("src_dir must be a string") + for key in ("exclude", "include", "gitignore"): + if key in project_config: + src_discovery_dict[key] = project_config[key] + if "comment_type" in project_config: + if project_config["comment_type"] not in SUPPORTED_COMMENT_TYPES: + src_discovery_errors.append( + f"comment_type must be one of {sorted(SUPPORTED_COMMENT_TYPES)}" + ) + else: + src_discovery_dict["file_types"] = list(SUPPORTED_COMMENT_TYPES) + return src_discovery_dict, src_discovery_errors diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py index 25da818..bec378b 100644 --- a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -238,8 +238,8 @@ def get_src_files( source_discover = SourceDiscover( dir_path, gitignore=src_trace_conf["gitignore"], - includes=src_trace_conf["include"], - excludes=src_trace_conf["exclude"], + include=src_trace_conf["include"], + exclude=src_trace_conf["exclude"], file_types=file_types, ) source_files.extend(source_discover.source_paths) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 411e1ab..957342e 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +import toml from typer.testing import CliRunner from sphinx_codelinks.cmd import app @@ -13,6 +14,27 @@ TEST_DIR, ) +ONELINE_COMMENT_TEMPLATE = { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + {"name": "id"}, + {"name": "title"}, + {"name": "type"}, + ], +} + +VDOC_CONFIG_TEMPLATE = { + "src_dir": str(TEST_DIR / "data" / "dcdc"), + "exclude": ["**/charge/demo_1.cpp", "**/discharge/demo_3.cpp"], + "include": ["**/charge/demo_2.cpp", "**/supercharge.cpp"], + "gitignore": True, + "file_types": ["cpp"], + "oneline_comment_style": ONELINE_COMMENT_TEMPLATE, +} + + runner = CliRunner() @@ -143,3 +165,70 @@ def test_vdoc(options, lines, tmp_path): assert result.exit_code == 0 assert result.stdout.splitlines() == lines + + +@pytest.mark.parametrize( + ("config_dict", "output"), + [ + ( + { + key: (123 if key == "exclude" else value) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Usage: root vdoc [OPTIONS]", + "Try 'root vdoc -h' for help.", + "╭─ Error ──────────────────────────────────────────────────────────────────────╮", + "│ Invalid value: Invalid source discovery configuration: │", + "│ Schema validation error in field 'exclude': 123 is not of type 'array' │", + "╰──────────────────────────────────────────────────────────────────────────────╯", + ], + ), + ( + { + key: (123 if key == "include" else value) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Usage: root vdoc [OPTIONS]", + "Try 'root vdoc -h' for help.", + "╭─ Error ──────────────────────────────────────────────────────────────────────╮", + "│ Invalid value: Invalid source discovery configuration: │", + "│ Schema validation error in field 'include': 123 is not of type 'array' │", + "╰──────────────────────────────────────────────────────────────────────────────╯", + ], + ), + ( + { + key: (123 if key in ("exclude", "include", "src_dir") else value) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Usage: root vdoc [OPTIONS]", + "Try 'root vdoc -h' for help.", + "╭─ Error ──────────────────────────────────────────────────────────────────────╮", + "│ Invalid value: Invalid source discovery configuration: │", + "│ src_dir must be a string │", + "│ Schema validation error in field 'exclude': 123 is not of type 'array' │", + "│ Schema validation error in field 'include': 123 is not of type 'array' │", + "╰──────────────────────────────────────────────────────────────────────────────╯", + ], + ), + ], +) +def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: + config_file = tmp_path / "vdoc_config.toml" + with config_file.open("w", encoding="utf-8") as f: + toml.dump(config_dict, f) + + options = [ + "vdoc", + "--config", + str(config_file), + ] + result = runner.invoke( + app, + options, + ) + stderr = result.stderr.splitlines() + assert stderr == output diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py index d74aa26..522d0b6 100644 --- a/tests/test_source_discover.py +++ b/tests/test_source_discover.py @@ -74,15 +74,15 @@ def test_source_discover_includes(source_directory: Path): source_discover = SourceDiscover( source_directory, gitignore=True, - excludes=["charge/*.cpp"], - includes=["**/*.cpp"], + exclude=["charge/*.cpp"], + include=["**/*.cpp"], ) assert len(source_discover.source_paths) == 5 def test_source_discover_excludes(source_directory: Path): source_discover = SourceDiscover( - source_directory, gitignore=True, excludes=["charge/*.cpp"] + source_directory, gitignore=True, exclude=["charge/*.cpp"] ) assert len(source_discover.source_paths) == 3 From d5bc060fd1cf390ae41850007355d24fe0cf1538 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 25 Jun 2025 10:44:17 +0200 Subject: [PATCH 14/54] fixed mypy --- src/sphinx_codelinks/cmd.py | 41 +++++++++++++------ .../sphinx_extension/config.py | 11 +++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index 95735ac..fbd2741 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -9,11 +9,15 @@ from sphinx_codelinks.source_discovery.config import SourceDiscoveryConfig from sphinx_codelinks.sphinx_extension.config import ( + SrcTraceProjectConfigFileType, SrcTraceProjectConfigType, build_src_discovery_dict, validate_oneline_comment_style, ) -from sphinx_codelinks.virtual_docs.config import OneLineCommentStyle +from sphinx_codelinks.virtual_docs.config import ( + OneLineCommentStyle, + OneLineCommentStyleType, +) app = typer.Typer( no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]} @@ -104,14 +108,33 @@ def vdoc( data = load_config_from_toml(config, project) errors: deque[str] = deque() - oneline_errors = validate_oneline_comment_style(data) + + oneline_comment_style_dict: OneLineCommentStyleType | None = data.get( + "oneline_comment_style" + ) + if oneline_comment_style_dict is None: + oneline_comment_style = OneLineCommentStyle() + else: + oneline_comment_style = OneLineCommentStyle(**oneline_comment_style_dict) + project_config = cast( + SrcTraceProjectConfigType, + { + key: value if key != "oneline_comment_style" else oneline_comment_style + for key, value in data.items() + if key != "oneline_comment_style" + }, + ) + oneline_errors = validate_oneline_comment_style(project_config) if oneline_errors: errors.appendleft("Invalid oneline comment style configuration:") errors.extend(oneline_errors) - src_discovery_dict, src_discovery_errors = build_src_discovery_dict(data) - src_discovery_config = SourceDiscoveryConfig(**src_discovery_dict) + src_discovery_dict, src_discovery_errors = build_src_discovery_dict(project_config) + if src_discovery_dict: + src_discovery_config = SourceDiscoveryConfig(**src_discovery_dict) + else: + src_discovery_config = SourceDiscoveryConfig() src_discovery_errors.extend(src_discovery_config.check_schema()) if src_discovery_errors: @@ -134,12 +157,6 @@ def vdoc( from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs - oneline_comment_style = data.get("oneline_comment_style") - if oneline_comment_style is None: - oneline_comment_style = OneLineCommentStyle() - else: - oneline_comment_style = OneLineCommentStyle(**oneline_comment_style) - virtual_docs = VirtualDocs( src_files=source_discover.source_paths, src_dir=str(src_root_dir), @@ -170,7 +187,7 @@ def vdoc( def load_config_from_toml( toml_file: Path, project: str | None = None -) -> SrcTraceProjectConfigType: +) -> SrcTraceProjectConfigFileType: try: with toml_file.open("rb") as f: toml_data = tomllib.load(f) @@ -183,7 +200,7 @@ def load_config_from_toml( f"Failed to load source tracing configuration from {toml_file}" ) from e - return cast(SrcTraceProjectConfigType, toml_data) + return cast(SrcTraceProjectConfigFileType, toml_data) if __name__ == "__main__": diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py index 3f1dcc7..8cdb075 100644 --- a/src/sphinx_codelinks/sphinx_extension/config.py +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -29,6 +29,17 @@ def __init__(self) -> None: file_lineno_href = SourceTracingLineHref() +class SrcTraceProjectConfigFileType(TypedDict): + # only support C/C++ for now + comment_type: Literal["cpp", "hpp", "c", "h"] + src_dir: str + remote_url_pattern: str + exclude: list[str] + include: list[str] + gitignore: bool + oneline_comment_style: OneLineCommentStyleType + + class SrcTraceProjectConfigType(TypedDict): # only support C/C++ for now comment_type: Literal["cpp", "hpp", "c", "h"] From edefdbe63a50f46d6f9c923509a72661ca38e998 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 25 Jun 2025 11:09:59 +0200 Subject: [PATCH 15/54] improve config validation --- src/sphinx_codelinks/cmd.py | 9 +++++++-- tests/test_cmd.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py index fbd2741..41290d8 100644 --- a/src/sphinx_codelinks/cmd.py +++ b/src/sphinx_codelinks/cmd.py @@ -115,13 +115,18 @@ def vdoc( if oneline_comment_style_dict is None: oneline_comment_style = OneLineCommentStyle() else: - oneline_comment_style = OneLineCommentStyle(**oneline_comment_style_dict) + try: + oneline_comment_style = OneLineCommentStyle(**oneline_comment_style_dict) + except TypeError as e: + raise typer.BadParameter( + f"Invalid oneline comment style configuration: {e}" + ) from e + project_config = cast( SrcTraceProjectConfigType, { key: value if key != "oneline_comment_style" else oneline_comment_style for key, value in data.items() - if key != "oneline_comment_style" }, ) oneline_errors = validate_oneline_comment_style(project_config) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 957342e..fbe3697 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -214,6 +214,42 @@ def test_vdoc(options, lines, tmp_path): "╰──────────────────────────────────────────────────────────────────────────────╯", ], ), + ( + { + key: ( + {"not_expected": 123} if key == "oneline_comment_style" else value + ) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Usage: root vdoc [OPTIONS]", + "Try 'root vdoc -h' for help.", + "╭─ Error ──────────────────────────────────────────────────────────────────────╮", + "│ Invalid value: Invalid oneline comment style configuration: │", + "│ OneLineCommentStyle.__init__() got an unexpected keyword argument │", + "│ 'not_expected' │", + "╰──────────────────────────────────────────────────────────────────────────────╯", + ], + ), + ( + { + key: ( + {"needs_fields": [{"name": "id"}, {"name": "id"}]} + if key == "oneline_comment_style" + else value + ) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Usage: root vdoc [OPTIONS]", + "Try 'root vdoc -h' for help.", + "╭─ Error ──────────────────────────────────────────────────────────────────────╮", + "│ Invalid value: Invalid oneline comment style configuration: │", + "│ Missing required fields: ['title', 'type'] │", + "│ Field 'id' is defined multiple times. │", + "╰──────────────────────────────────────────────────────────────────────────────╯", + ], + ), ], ) def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: From 4b0bf7a1dd2a8b8ee069f8a545eb5c0da60247ed Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 25 Jun 2025 12:18:59 +0200 Subject: [PATCH 16/54] disabled color for testing --- tests/test_cmd.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index fbe3697..9983217 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -262,9 +262,6 @@ def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: "--config", str(config_file), ] - result = runner.invoke( - app, - options, - ) + result = runner.invoke(app, options, color=False) stderr = result.stderr.splitlines() assert stderr == output From 97a2c19a2774405646454a4f2b583602f0b5eeba Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 25 Jun 2025 12:39:38 +0200 Subject: [PATCH 17/54] strip ANSI --- tests/test_cmd.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 9983217..d8b7586 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -1,4 +1,5 @@ from pathlib import Path +import re import pytest import toml @@ -263,5 +264,9 @@ def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: str(config_file), ] result = runner.invoke(app, options, color=False) - stderr = result.stderr.splitlines() + stderr = strip_ansi(result.stderr).splitlines() assert stderr == output + + +def strip_ansi(text): + return re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text) From 85e1592ca4f9f003745dd11fbaeb8a14249b8101 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 25 Jun 2025 12:52:42 +0200 Subject: [PATCH 18/54] set NO_COLOR --- tests/test_cmd.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index d8b7586..01094a2 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -1,6 +1,6 @@ from pathlib import Path -import re +from _pytest.monkeypatch import MonkeyPatch import pytest import toml from typer.testing import CliRunner @@ -254,6 +254,9 @@ def test_vdoc(options, lines, tmp_path): ], ) def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: + # Force disable Rich styling + monkeypatch = MonkeyPatch() + monkeypatch.setenv("NO_COLOR", "1") config_file = tmp_path / "vdoc_config.toml" with config_file.open("w", encoding="utf-8") as f: toml.dump(config_dict, f) @@ -264,9 +267,5 @@ def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: str(config_file), ] result = runner.invoke(app, options, color=False) - stderr = strip_ansi(result.stderr).splitlines() + stderr = result.stderr.splitlines() assert stderr == output - - -def strip_ansi(text): - return re.sub(r"\x1b\[[0-?]*[ -/]*[@-~]", "", text) From 9a57ed3ba742443761a7eac82a074f068154721c Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 25 Jun 2025 12:55:42 +0200 Subject: [PATCH 19/54] removed first two lines --- tests/test_cmd.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 01094a2..bf9317c 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -177,8 +177,6 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "Usage: root vdoc [OPTIONS]", - "Try 'root vdoc -h' for help.", "╭─ Error ──────────────────────────────────────────────────────────────────────╮", "│ Invalid value: Invalid source discovery configuration: │", "│ Schema validation error in field 'exclude': 123 is not of type 'array' │", @@ -191,8 +189,6 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "Usage: root vdoc [OPTIONS]", - "Try 'root vdoc -h' for help.", "╭─ Error ──────────────────────────────────────────────────────────────────────╮", "│ Invalid value: Invalid source discovery configuration: │", "│ Schema validation error in field 'include': 123 is not of type 'array' │", @@ -205,8 +201,6 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "Usage: root vdoc [OPTIONS]", - "Try 'root vdoc -h' for help.", "╭─ Error ──────────────────────────────────────────────────────────────────────╮", "│ Invalid value: Invalid source discovery configuration: │", "│ src_dir must be a string │", @@ -223,8 +217,6 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "Usage: root vdoc [OPTIONS]", - "Try 'root vdoc -h' for help.", "╭─ Error ──────────────────────────────────────────────────────────────────────╮", "│ Invalid value: Invalid oneline comment style configuration: │", "│ OneLineCommentStyle.__init__() got an unexpected keyword argument │", @@ -242,8 +234,6 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "Usage: root vdoc [OPTIONS]", - "Try 'root vdoc -h' for help.", "╭─ Error ──────────────────────────────────────────────────────────────────────╮", "│ Invalid value: Invalid oneline comment style configuration: │", "│ Missing required fields: ['title', 'type'] │", @@ -267,5 +257,5 @@ def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: str(config_file), ] result = runner.invoke(app, options, color=False) - stderr = result.stderr.splitlines() + stderr = result.stderr.splitlines()[2:] assert stderr == output From bdc569d055cd27b81311bb07e11b0fa7fe209215 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 25 Jun 2025 13:10:17 +0200 Subject: [PATCH 20/54] ignore format from typer --- tests/test_cmd.py | 52 ++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index bf9317c..fa31858 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -1,6 +1,5 @@ from pathlib import Path -from _pytest.monkeypatch import MonkeyPatch import pytest import toml from typer.testing import CliRunner @@ -169,7 +168,7 @@ def test_vdoc(options, lines, tmp_path): @pytest.mark.parametrize( - ("config_dict", "output"), + ("config_dict", "output_lines"), [ ( { @@ -177,10 +176,8 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "╭─ Error ──────────────────────────────────────────────────────────────────────╮", - "│ Invalid value: Invalid source discovery configuration: │", - "│ Schema validation error in field 'exclude': 123 is not of type 'array' │", - "╰──────────────────────────────────────────────────────────────────────────────╯", + "Invalid value: Invalid source discovery configuration:", + "Schema validation error in field 'exclude': 123 is not of type 'array'", ], ), ( @@ -189,10 +186,8 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "╭─ Error ──────────────────────────────────────────────────────────────────────╮", - "│ Invalid value: Invalid source discovery configuration: │", - "│ Schema validation error in field 'include': 123 is not of type 'array' │", - "╰──────────────────────────────────────────────────────────────────────────────╯", + "Invalid value: Invalid source discovery configuration:", + "Schema validation error in field 'include': 123 is not of type 'array'", ], ), ( @@ -201,12 +196,10 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "╭─ Error ──────────────────────────────────────────────────────────────────────╮", - "│ Invalid value: Invalid source discovery configuration: │", - "│ src_dir must be a string │", - "│ Schema validation error in field 'exclude': 123 is not of type 'array' │", - "│ Schema validation error in field 'include': 123 is not of type 'array' │", - "╰──────────────────────────────────────────────────────────────────────────────╯", + "Invalid value: Invalid source discovery configuration:", + "src_dir must be a string", + "Schema validation error in field 'exclude': 123 is not of type 'array'", + "Schema validation error in field 'include': 123 is not of type 'array'", ], ), ( @@ -217,11 +210,9 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "╭─ Error ──────────────────────────────────────────────────────────────────────╮", - "│ Invalid value: Invalid oneline comment style configuration: │", - "│ OneLineCommentStyle.__init__() got an unexpected keyword argument │", - "│ 'not_expected' │", - "╰──────────────────────────────────────────────────────────────────────────────╯", + "Invalid value: Invalid oneline comment style configuration:", + "OneLineCommentStyle.__init__() got an unexpected keyword argument", + "'not_expected'", ], ), ( @@ -234,19 +225,15 @@ def test_vdoc(options, lines, tmp_path): for key, value in VDOC_CONFIG_TEMPLATE.items() }, [ - "╭─ Error ──────────────────────────────────────────────────────────────────────╮", - "│ Invalid value: Invalid oneline comment style configuration: │", - "│ Missing required fields: ['title', 'type'] │", - "│ Field 'id' is defined multiple times. │", - "╰──────────────────────────────────────────────────────────────────────────────╯", + "Invalid value: Invalid oneline comment style configuration:", + "Missing required fields: ['title', 'type']", + "Field 'id' is defined multiple times.", ], ), ], ) -def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: +def test_vdoc_config_negative(config_dict, output_lines, tmp_path: Path) -> None: # Force disable Rich styling - monkeypatch = MonkeyPatch() - monkeypatch.setenv("NO_COLOR", "1") config_file = tmp_path / "vdoc_config.toml" with config_file.open("w", encoding="utf-8") as f: toml.dump(config_dict, f) @@ -256,6 +243,7 @@ def test_vdoc_config_negative(config_dict, output, tmp_path: Path) -> None: "--config", str(config_file), ] - result = runner.invoke(app, options, color=False) - stderr = result.stderr.splitlines()[2:] - assert stderr == output + result = runner.invoke(app, options) + assert result.exit_code != 0 + for line in output_lines: + assert line in result.stderr From 3fb60a2c3fc67fc92011470153ff698df3d99a16 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 25 Jun 2025 16:06:35 +0200 Subject: [PATCH 21/54] improved docs --- docs/conf.py | 10 +++--- docs/source/configuration.rst | 15 ++++++++ docs/source/index.rst | 67 ++++++----------------------------- docs/source/usage.rst | 40 +++++++++++++++++++++ 4 files changed, 71 insertions(+), 61 deletions(-) create mode 100644 docs/source/configuration.rst create mode 100644 docs/source/usage.rst diff --git a/docs/conf.py b/docs/conf.py index 304fe0d..7f4865d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,10 +22,7 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = [ - "sphinx_needs", - "sphinx_codelinks", -] +extensions = ["sphinx_needs", "sphinx_codelinks", "sphinx.ext.intersphinx"] # exclude_patterns = [] templates_path = ["_templates"] @@ -33,6 +30,11 @@ todo_include_todos = True +# -- Options for intersphinx extension --------------------------------------- + +intersphinx_mapping = { + "needs": ("https://sphinx-needs.readthedocs.io/en/latest/", None), +} # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst new file mode 100644 index 0000000..e0298b3 --- /dev/null +++ b/docs/source/configuration.rst @@ -0,0 +1,15 @@ +Configuration +------------- + +The config for source tracing can be specified in ``conf.py`` or ``toml`` file. +In the case where the config is introduced in ``toml`` file, the config path needs to be specified in ``conf.py`` + +.. code-block:: python + + # Specify the config path for source tracing in conf.py + src_trace_config_from_toml = "src_trace.toml" + +**Example Config** + +.. literalinclude:: ./../src_trace.toml + :language: toml diff --git a/docs/source/index.rst b/docs/source/index.rst index a16aeb1..22239d2 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,61 +1,14 @@ -CodeLinks -========= +Introduction +============ -``CodeLinks`` is a sphinx extension that provides a directive ``src-trace`` to trace the needs of source files. +``CodeLinks`` is a sphinx extension that provides a directive ``src-trace`` to trace the :external+needs:doc:`needs ` defined in source files. -Usage ------ +Contents +-------- -.. code-block:: rst +.. toctree:: + :maxdepth: 2 + :caption: Basics - .. src-trace:: example_with_file - :project: project_config - :file: example.cpp - - -or - -.. code-block:: rst - - .. src-trace:: example_with_directory - :project: project_config - :directory: ./example - - -``src-trace`` directive has the following options: - -* **project**: the project config specified in ``conf.py`` or ``toml`` to be used for source tracing. -* **file**: the source file to be traced. -* **directory**: the source files in the directory to be traced recursively. - -Regarding the options **file** and **directory**: - -- they are optional and mutually exclusive. -- the given paths of them are relative to ``src_dir`` defined in the source tracing configuration -- if not given, the whole project will be examined. - -**Example** - -.. src-trace:: dcdc demo_1 - :project: dcdc - :file: ./charge/demo_1.cpp - -.. src-trace:: dcdc charge - :project: dcdc - :directory: ./discharge - -Config ------- - -The config for source tracing can be specified in ``conf.py`` or ``toml`` file. -In the case where the config is introduced in ``toml`` file, the config path needs to be specified in ``conf.py`` - -.. code-block:: python - - # Specify the config path for source tracing in conf.py - src_trace_config_from_toml = "src_trace.toml" - -**Example Config** - -.. literalinclude:: ./../src_trace.toml - :language: toml + usage + configuration diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 0000000..7e2db8f --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,40 @@ +Usage +===== + +.. code-block:: rst + + .. src-trace:: example_with_file + :project: project_config + :file: example.cpp + + +or + +.. code-block:: rst + + .. src-trace:: example_with_directory + :project: project_config + :directory: ./example + + +``src-trace`` directive has the following options: + +* **project**: the project config specified in ``conf.py`` or ``toml`` to be used for source tracing. +* **file**: the source file to be traced. +* **directory**: the source files in the directory to be traced recursively. + +Regarding the options **file** and **directory**: + +- they are optional and mutually exclusive. +- the given paths of them are relative to ``src_dir`` defined in the source tracing configuration +- if not given, the whole project will be examined. + +**Example** + +.. src-trace:: dcdc demo_1 + :project: dcdc + :file: ./charge/demo_1.cpp + +.. src-trace:: dcdc charge + :project: dcdc + :directory: ./discharge From 1988da705a756b0aaa62c5723e72506ff10ce955 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Mon, 7 Jul 2025 10:19:09 +0200 Subject: [PATCH 22/54] updated pyproject --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 26af91a..2b8c0a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "typer>=0.16.0", "jsonschema", "sphinx>=7.4,<9", + "sphinx-needs>=4.2.0", # unconstrained versions, to be pinned by user or Sphinx "jinja2", "pygments", @@ -42,7 +43,6 @@ dev-dependencies = [ "pytest>=8.2.2", "simple-build>=0.0.2", "sphinx-design>=0.5.0", - "sphinx-needs>=4.2.0", "types-psutil>=7.0.0.20250218", "uv>=0.5.5", "pytest-docker>=3.1.2", @@ -62,9 +62,9 @@ cl = "sphinx_codelinks.cmd:app" "rye:format" = "rye format" "check" = { chain = ["rye:format", "rye:lint", "mypy:all"] } # docs html -"docs_html:rm" = "rm -rf docs/_build/html" -"docs_html" = "sphinx-build -nW --keep-going -b html -T -c docs docs/source docs/_build/html" -"docs_html:clean" = { chain = ["docs_html:rm", "docs_html"] } +"docs:rm" = "rm -rf docs/_build/html" +"docs" = "sphinx-build -nW --keep-going -b html -T -c docs docs/source docs/_build/html" +"docs:clean" = { chain = ["docs:rm", "docs"] } # pytest prod "pytest:prod" = { cmd = "uv run --with-requirements requirements.lock --no-editable --refresh pytest tests/" } From 2b8dc69a4012ea98353f380fd1b3290be9950274 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Mon, 7 Jul 2025 10:24:00 +0200 Subject: [PATCH 23/54] added installation doc --- docs/source/basics/installation.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/source/basics/installation.rst diff --git a/docs/source/basics/installation.rst b/docs/source/basics/installation.rst new file mode 100644 index 0000000..1c9c46b --- /dev/null +++ b/docs/source/basics/installation.rst @@ -0,0 +1,23 @@ +.. _installation: + +Installation +============ + +Using Pip +---------- + +.. code-block:: bash + + pip install sphinx-codelinks + +Activation +---------- + +For activation, please add `sphinx_needs` and `sphinx-codelinks` to the projects's extension list of your **conf.py** file + +.. code-block:: python + + extensions = [ + 'sphinx_needs', + 'sphinx_codelinks' + ] From b8e60effec930c54853bf83a335b8c3a373c0e02 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Mon, 7 Jul 2025 15:46:37 +0200 Subject: [PATCH 24/54] update docs --- docs/conf.py | 9 +- docs/source/basics/installation.rst | 2 +- docs/source/basics/introduction.rst | 14 ++ docs/source/{ => basics}/usage.rst | 0 docs/source/configuration.rst | 351 +++++++++++++++++++++++++++- docs/source/index.rst | 54 ++++- 6 files changed, 419 insertions(+), 11 deletions(-) create mode 100644 docs/source/basics/introduction.rst rename docs/source/{ => basics}/usage.rst (100%) diff --git a/docs/conf.py b/docs/conf.py index 7f4865d..883ad79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,13 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ["sphinx_needs", "sphinx_codelinks", "sphinx.ext.intersphinx"] +extensions = [ + "sphinx_design", + "sphinx_needs", + "sphinx_codelinks", + "sphinx.ext.intersphinx", + "sphinx_code_tabs", +] # exclude_patterns = [] templates_path = ["_templates"] @@ -34,6 +40,7 @@ intersphinx_mapping = { "needs": ("https://sphinx-needs.readthedocs.io/en/latest/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master", None), } # -- Options for HTML output ------------------------------------------------- diff --git a/docs/source/basics/installation.rst b/docs/source/basics/installation.rst index 1c9c46b..0f3fcc9 100644 --- a/docs/source/basics/installation.rst +++ b/docs/source/basics/installation.rst @@ -13,7 +13,7 @@ Using Pip Activation ---------- -For activation, please add `sphinx_needs` and `sphinx-codelinks` to the projects's extension list of your **conf.py** file +For activation, please add ``sphinx_needs`` and ``sphinx-codelinks`` to the projects's extension list of your **conf.py** file .. code-block:: python diff --git a/docs/source/basics/introduction.rst b/docs/source/basics/introduction.rst new file mode 100644 index 0000000..17164d3 --- /dev/null +++ b/docs/source/basics/introduction.rst @@ -0,0 +1,14 @@ +Introduction +============ + +``CodeLinks`` is a sphinx extension that provides a directive ``src-trace`` +to trace the :external+needs:doc:`Sphinx-Needs ` defined in source files. + +The provided directive leverages the other two modules ``SourceDiscovery`` and ``VirtualDocs``, +which are also packed in the extension, +to discover source files and create the virtual documents for ``src-trace`` to consume. + +Both ``SourceDiscovery`` and ``VirtualDocs`` provide the followings for the developers : + +- **Python API** to extend other further use cases. +- **CLI** to have atomic steps in CI/CD pipelines. diff --git a/docs/source/usage.rst b/docs/source/basics/usage.rst similarity index 100% rename from docs/source/usage.rst rename to docs/source/basics/usage.rst diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e0298b3..a73c6b7 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -1,15 +1,358 @@ Configuration -------------- +============= -The config for source tracing can be specified in ``conf.py`` or ``toml`` file. -In the case where the config is introduced in ``toml`` file, the config path needs to be specified in ``conf.py`` +The configuration for ``CodeLinks`` take place in the project's :external+sphinx:ref:`conf.py file `. + +Each source code project may have different configurations because of its programming language or its locations. +Therefore, based on such consideration, there are **global options** and **project-specific options** for ``CodeLinks`` + +All configuration options starts with the prefix ``src_trace_`` for **Sphinx-CodeLinks**. + +Global Options +-------------- + +The options starts with the prefix ``src_trace_`` are globally applied in the scope of Sphinx documentation. + +src_trace_config_from_toml +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This configuration takes the (relative) path to a `toml file `__ which contains some or all of the ``CodeLinks`` configuration (configuration in the toml will override that in the :file:`conf.py`). .. code-block:: python # Specify the config path for source tracing in conf.py src_trace_config_from_toml = "src_trace.toml" -**Example Config** +Configuration in the toml can contain any of the following options, under a ``[src_trace]`` section, +but with the ``src_trace_`` prefix removed. + +For example: .. literalinclude:: ./../src_trace.toml :language: toml + +.. caution:: Any configuration specifying relative paths in the toml file will be resolved relatively to the directory containing the :file:`conf.py` file. + +.. _`src_trace_set_local_url`: + +src_trace_set_local_url +~~~~~~~~~~~~~~~~~~~~~~~ + +Set this option to ``False``, if the local link between a need to the local source code where it is defined is not required. + +Default: **True** + +.. tabs:: + + .. code-tab:: python + + src_trace_set_local_url = True + + .. code-tab:: toml + + [src_trace] + set_local_url = true + +src_trace_set_local_field +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This option is only optionally required, if :ref:`src_trace_set_local_url` is set to **True**. + +Set the desired custom field name for the local link to the source code. + +Default: **local-url** + +.. tabs:: + + .. code-tab:: python + + src_trace_local_url_field = "local-url" + + .. code-tab:: toml + + [src_trace] + local_url_field = "local-url" + +.. _`src_trace_set_remote_url`: + +src_trace_set_remote_url +~~~~~~~~~~~~~~~~~~~~~~~~ + +Set this option to ``False``, if the remote link between a need to the remote source code +where it is defined is not required. + +The remote means where the source code is hosted such as GitHub. + +Default: **True** + +.. tabs:: + + .. code-tab:: python + + src_trace_set_remote_url = True + + .. code-tab:: toml + + [src_trace] + set_remote_url = true + +src_trace_set_remote_field +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This option is only optionally required, if :ref:`src_trace_set_remote_url` is set to **True**. + +Set the desired custom field name for the remote link to the source code. + +Default: **remote-url** + +.. tabs:: + + .. code-tab:: python + + src_trace_remote_url_field = "remote-url" + + .. code-tab:: toml + + [src_trace] + remote_url_field = "remote-url" + +Project Specific Options +------------------------ + +Options defined in **src_trace_projects** are project-specific. + +src_trace_projects +~~~~~~~~~~~~~~~~~~ + +This option contains multiple sets of project-specific options. The project name is defined as the key in a dictionary +and its corresponding value is a dictionary containing the options specific to that project. + +.. tabs:: + + .. code-tab:: python + + project_options = dict() + src_trace_projects = { + "project_name": project_options + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + # Project configuration for "project_name" shall be written here + +comment_type +~~~~~~~~~~~~ + +The option defined the comment type used in source code of the project. + +Default: **cpp** + +.. note:: Currently, only C/C++ is supported + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "comment_type": "c" + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + comment_type = "c" + +src_dir +~~~~~~~ + +The relative path based on ``conf.py`` file (NOT SURE) to the source code's root directory + +Default: **./** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "src_dir": "./../src" + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + src_dir = "./../src" + +remote_url_pattern +~~~~~~~~~~~~~~~~~~ + +This option only works with :ref:`src_trace_set_remote_url` set to **True**. +The pattern to access the source code to the remote repositories such as GitHub. + +Default: **Not set** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" + +This option leverages the configuration of :external+needs:ref:`need_string_links` +with the following setup: + +.. code-block:: python + + remote_url_pattern = remote_url_pattern.format( + commit=commit_id, + path=f"{remote_src_dir}/" + "{{value}}", + line="{{lineno}}", + ) + + { + "regex": r"^(?P.+)#L(?P.*)?", + "link_url": remote_url_pattern, + "link_name": "{{value}}#L{{lineno}}", + "options": [remote_url_field], + } + +exclude +~~~~~~~ + +The option is a list of glob patterns to exclude the files which are not required to be addressed + +Default: **[]** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "exclude": ["dcdc/src/ubt/ubt.cpp"] + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + exclude = ["dcdc/src/ubt/ubt.cpp"] + +include +~~~~~~~ + +The option is a list of glob patterns to include the files which are required to be addressed + +Default: **[]** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = + { + "project_name": { + "include": ["dcdc/src/ubt/ubt.cpp"] + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + include = ["dcdc/src/ubt/ubt.cpp"] + +.. note:: **include** option has the highest priority over **exclude** and **gitignore** options. + +gitignore +~~~~~~~~~ + +The option to respect .gitignore :file + +Default: **True** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "gitignore": False + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + gitignore = false + +.. attention:: The option currently do NOT support nested .gitignore + +oneline_comment_style +~~~~~~~~~~~~~~~~~~~~~ + +This option enables users to simply define a customized one-line-pattern comment to represent +``Sphinx-Needs`` instead of using RST. + +Default: + +.. tabs:: + + .. code-tab:: python + + import os + src_trace_projects = { + "project_name": { + "oneline_comment_style": { + "start_sequence": "@", + "end_sequence": os.linesep, + "field_split_char": ",", + needs_fields = [ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ] + } + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name.oneline_comment_style] + start_sequence = "@" + # end_sequence for the online comments; default is an os-dependant newline character + field_split_char = "," + needs_fields = [ + { "name" = "title", "type" = "str" }, + { "name" = "id", "type" = "str" }, + { "name" = "type", "type" = "str", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [] }, + ] + +With the default, the following one-line comment will be extracted by ``CodeLinks`` and +it is equivalent to the following RST + +.. tabs:: + + .. code-tab:: c + + // @Function Bar, IMPL_4, impl, [SPEC_1, SPEC_2] + + .. code-tab:: RST + + .. impl:: Function Bar + :id: IMPL_4 + :links: [SPEC_1, SPEC_2] + +More uses cases can be found in `tests `__ diff --git a/docs/source/index.rst b/docs/source/index.rst index 22239d2..1f4919b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,14 +1,58 @@ -Introduction -============ -``CodeLinks`` is a sphinx extension that provides a directive ``src-trace`` to trace the :external+needs:doc:`needs ` defined in source files. + +.. grid:: + :class-row: sd-w-100 + + .. grid-item:: + :columns: 12 8 8 8 + :child-align: justify + :class: sd-fs-3 + + .. div:: sd-font-weight-bold + + The portal to your source code + + .. div:: sd-fs-5 sd-font-italic + + + ``Sphinx-CodeLinks`` is designed for Engineering-as-Code workflow to facilitate ALM. + It enables users to defined ``Sphinx-Needs`` within source code and automatically extract them + into the documentation during the Sphinx build process. + + .. grid:: 1 1 2 2 + :gutter: 2 2 3 3 + :margin: 2 + :padding: 0 + + .. grid-item:: + :columns: auto + + .. button-ref:: basics/installation + :ref-type: doc + :outline: + :color: primary + :class: sd-rounded-pill sd-px-4 sd-fs-5 + + Get Started + + .. grid-item:: + :columns: auto + + .. button-link:: https://useblocks.com/ + :outline: + :color: primary + :class: sd-rounded-pill sd-px-4 sd-fs-5 + + About useblocks Contents -------- .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: Basics - usage + basics/introduction + basics/installation + basics/usage configuration From 9d795982e846281e52acf9cfad25af81a5232834 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Tue, 8 Jul 2025 13:46:28 +0200 Subject: [PATCH 25/54] add development docs --- docs/source/development/change_log.rst | 17 +++++++++++++++++ docs/source/development/roadmap.rst | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 docs/source/development/change_log.rst create mode 100644 docs/source/development/roadmap.rst diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst new file mode 100644 index 0000000..ce4f10b --- /dev/null +++ b/docs/source/development/change_log.rst @@ -0,0 +1,17 @@ +.. _changelog: + +Changelog +========= + +0.1.0 +----- + +:Released: 11.07.2025 + +Initial release of ``Sphinx-CodeLinks`` + +This version features: + +- Sphinx Directive ``src-trace`` +- Virtual Docs and Source Discovery CLI +- One-line comment to define ``Sphinx-Needs`` diff --git a/docs/source/development/roadmap.rst b/docs/source/development/roadmap.rst new file mode 100644 index 0000000..1d11478 --- /dev/null +++ b/docs/source/development/roadmap.rst @@ -0,0 +1,19 @@ +.. _roadmap: + +Roadmap +======= + +Other Comment styles +-------------------- + +Currently, only ``C/C++`` comment style is supported. +The other comment styles for different programming languages are planed, such as: + +- Python +- Rust + +Nested .gitignore +----------------- + +``CodeLinks`` respects ``.gitignore`` file, but if the .gitignore files are nested, it's not supported. +Respecting nested ``.gitignore`` in the context of the git repositories is planned. From 790050aacc480d149dcd490b62c45fcbdbf154e7 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Tue, 8 Jul 2025 13:47:05 +0200 Subject: [PATCH 26/54] changed layout of docs --- docs/source/basics/{usage.rst => quickstart.rst} | 0 docs/source/components/cli.rst | 0 docs/source/{ => components}/configuration.rst | 0 docs/source/index.rst | 16 ++++++++++++++-- 4 files changed, 14 insertions(+), 2 deletions(-) rename docs/source/basics/{usage.rst => quickstart.rst} (100%) create mode 100644 docs/source/components/cli.rst rename docs/source/{ => components}/configuration.rst (100%) diff --git a/docs/source/basics/usage.rst b/docs/source/basics/quickstart.rst similarity index 100% rename from docs/source/basics/usage.rst rename to docs/source/basics/quickstart.rst diff --git a/docs/source/components/cli.rst b/docs/source/components/cli.rst new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/configuration.rst b/docs/source/components/configuration.rst similarity index 100% rename from docs/source/configuration.rst rename to docs/source/components/configuration.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 1f4919b..f880e99 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -54,5 +54,17 @@ Contents basics/introduction basics/installation - basics/usage - configuration + basics/quickstart + +.. toctree:: + :maxdepth: 1 + :caption: Components + + components/configuration + +.. toctree:: + :maxdepth: 1 + :caption: Development + + development/roadmap + development/change_log From c3524aa515d5a4c26107d4994e960342383cb731 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Tue, 8 Jul 2025 14:00:12 +0200 Subject: [PATCH 27/54] added CLI docs --- docs/source/basics/quickstart.rst | 4 ++-- docs/source/components/cli.rst | 15 +++++++++++++++ docs/source/components/configuration.rst | 2 +- docs/source/index.rst | 1 + 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/source/basics/quickstart.rst b/docs/source/basics/quickstart.rst index 7e2db8f..f341960 100644 --- a/docs/source/basics/quickstart.rst +++ b/docs/source/basics/quickstart.rst @@ -1,5 +1,5 @@ -Usage -===== +Quick Start +=========== .. code-block:: rst diff --git a/docs/source/components/cli.rst b/docs/source/components/cli.rst index e69de29..52de229 100644 --- a/docs/source/components/cli.rst +++ b/docs/source/components/cli.rst @@ -0,0 +1,15 @@ +Command Line Interface (CLI) +============================ + +``Sphinx-CodeLinks`` provides CLI for users to integrate documentation build into CI/CD pipeline +and for local development. + +It features help pages. add ``-h`` or ``--help`` to any command to see the available options. + +.. typer:: sphinx_codelinks.cmd.app + :prog: codelinks + :width: 85 + :preferred: svg + :theme: monokai + :show-nested: + :make-sections: diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index a73c6b7..1473d58 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -28,7 +28,7 @@ but with the ``src_trace_`` prefix removed. For example: -.. literalinclude:: ./../src_trace.toml +.. literalinclude:: ./../../src_trace.toml :language: toml .. caution:: Any configuration specifying relative paths in the toml file will be resolved relatively to the directory containing the :file:`conf.py` file. diff --git a/docs/source/index.rst b/docs/source/index.rst index f880e99..46d5d95 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -61,6 +61,7 @@ Contents :caption: Components components/configuration + components/cli .. toctree:: :maxdepth: 1 From 57b1ac04edff71bcbd3270229dcee210c880a043 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Tue, 8 Jul 2025 14:04:22 +0200 Subject: [PATCH 28/54] added typer ext --- docs/conf.py | 1 + pyproject.toml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 883ad79..87494b8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,6 +28,7 @@ "sphinx_codelinks", "sphinx.ext.intersphinx", "sphinx_code_tabs", + "sphinxcontrib.typer", ] # exclude_patterns = [] diff --git a/pyproject.toml b/pyproject.toml index 2b8c0a6..5cfbec3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dev-dependencies = [ "pytest-cov>=5.0.0", "pytest>=8.2.2", "simple-build>=0.0.2", - "sphinx-design>=0.5.0", + "sphinx-design>=0.6.1", "types-psutil>=7.0.0.20250218", "uv>=0.5.5", "pytest-docker>=3.1.2", @@ -50,10 +50,12 @@ dev-dependencies = [ "insta-science>=0.2.1", "types-jsonschema>=4.23.0.20241208", "toml>=0.10.2", + "sphinx-code-tabs>=0.5.5", + "sphinxcontrib-typer>=0.5.1", ] [project.scripts] -cl = "sphinx_codelinks.cmd:app" +codelinks = "sphinx_codelinks.cmd:app" [tool.rye.scripts] # linting and formatting From b5c0981d6d103bca52012e1cf41e68c0cb13ed6e Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Tue, 8 Jul 2025 14:15:07 +0200 Subject: [PATCH 29/54] updated build cmd --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0c727b..ef7dcdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,7 @@ jobs: run: sudo apt-get --yes install graphviz - uses: ./.github/actions/setup_rye - name: Run documentation build - run: rye run docs_html + run: rye run docs all_good: # This job does nothing and is only used for the branch protection From 1b8ff12f1fd147055de541bda62a2d5bde368766 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Tue, 8 Jul 2025 15:41:46 +0200 Subject: [PATCH 30/54] added contributing docs --- docs/source/development/contributing.rst | 73 ++++++++++++++++++++++++ docs/source/index.rst | 1 + 2 files changed, 74 insertions(+) create mode 100644 docs/source/development/contributing.rst diff --git a/docs/source/development/contributing.rst b/docs/source/development/contributing.rst new file mode 100644 index 0000000..b00b72f --- /dev/null +++ b/docs/source/development/contributing.rst @@ -0,0 +1,73 @@ +Contributing +============ + +This page provides a guide for developers wishing to contribute to ``Sphinx-CodeLinks``. + +Bugs, Features and PRs +---------------------- + +For **bug reports** and well-described **technical feature request**, please use our issue tracker: +https://github.com/useblocks/sphinx-codelinks/issues + +If you have already created a PR, you can send it in. Our CI workflow will check (test and code styles) +and a maintainer will perform a review, before we can merge it. +Your PR should conform with the following rules: + +- A meaningful description or link, which describes the change +- The changed code (for sure :) ) +- Test cases for the change (important!) +- Updated documentation, if behavior gets changed or new options/directives are introduced. +- Update of docs/changelog.rst. + +Install Dependencies +-------------------- + +``CodeLinks`` uses `rye `_ to manage the repository. + +For the development, use the following command to install python dependencies into the virtual environment. + +.. code-block:: bash + + rye sync + +Formatting, Linting and Typing +------------------------------ + +To run the formatting and linting, pre-commit is used: + +.. code-block:: bash + + pre-commit install # to auto-run on every commit + pre-commit run --all-files # to run manually + + +The CI also checks typing, use the following command locally to see if your code is well-typed + +.. code-block:: bash + + rye run mypy:all + +Build docs +---------- + +To build the documentation stored in ``docs``, run: + +.. code-block:: bash + + rye run docs + +Test Cases +---------- + +To run test cases locally: + +.. code-block:: bash + + rye run pytest:prod + +Note some tests use `syrupy `__ to perform snapshot testing. +These snapshots can be updated by running: + +.. code-block:: bash + + pytest tests/ --snapshot-update diff --git a/docs/source/index.rst b/docs/source/index.rst index 46d5d95..0e8d336 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -69,3 +69,4 @@ Contents development/roadmap development/change_log + development/contributing From 33eedf88073ffcdaa233b8ecc92482baf2f27029 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Tue, 8 Jul 2025 17:17:34 +0200 Subject: [PATCH 31/54] update docs --- docs/source/basics/introduction.rst | 4 ++ docs/source/basics/quickstart.rst | 38 ++++++++++- docs/source/components/configuration.rst | 13 ++-- docs/source/components/oneline.rst | 83 ++++++++++++++++++++++++ docs/source/development/roadmap.rst | 14 ++++ 5 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 docs/source/components/oneline.rst diff --git a/docs/source/basics/introduction.rst b/docs/source/basics/introduction.rst index 17164d3..e83f94f 100644 --- a/docs/source/basics/introduction.rst +++ b/docs/source/basics/introduction.rst @@ -4,6 +4,10 @@ Introduction ``CodeLinks`` is a sphinx extension that provides a directive ``src-trace`` to trace the :external+needs:doc:`Sphinx-Needs ` defined in source files. +Instead of putting RST syntax in the comment, the need definition in source code is simplified to one-liner only, +so that users can just write their `customized one-line comment `_ to have the traceability +from the link between source code and documentation. + The provided directive leverages the other two modules ``SourceDiscovery`` and ``VirtualDocs``, which are also packed in the extension, to discover source files and create the virtual documents for ``src-trace`` to consume. diff --git a/docs/source/basics/quickstart.rst b/docs/source/basics/quickstart.rst index f341960..971fc2d 100644 --- a/docs/source/basics/quickstart.rst +++ b/docs/source/basics/quickstart.rst @@ -1,6 +1,8 @@ Quick Start =========== +``CodeLinks`` provides ``src-trace`` directive and it can be used in the following ways: + .. code-block:: rst .. src-trace:: example_with_file @@ -29,12 +31,46 @@ Regarding the options **file** and **directory**: - the given paths of them are relative to ``src_dir`` defined in the source tracing configuration - if not given, the whole project will be examined. -**Example** +Example +------- + +With the following configuration for a demo source code project `dcdc `_, + +.. code-block:: python + :caption: conf.py + + src_trace_config_from_toml = "src_trace.toml" + +.. literalinclude:: ./../../src_trace.toml + :caption: src_trace.toml + :language: toml + +``src-trace`` directive can be used with **file** option: + +.. code-block:: rst + + .. src-trace:: dcdc demo_1 + :project: dcdc + :file: ./charge/demo_1.cpp + +The needs defined in source code are extracted and rendered to: .. src-trace:: dcdc demo_1 :project: dcdc :file: ./charge/demo_1.cpp +``src-trace`` directive can be used with **directory** option: + +.. code-block:: rst + + .. src-trace:: dcdc charge + :project: dcdc + :directory: ./discharge + +The needs defined in source code are extracted and rendered to: + .. src-trace:: dcdc charge :project: dcdc :directory: ./discharge + +To have more customized configuration of ``CodeLinks``, please refer to :ref:`configuration ` diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 1473d58..1dc18aa 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -1,3 +1,5 @@ +.. _configuration: + Configuration ============= @@ -16,7 +18,9 @@ The options starts with the prefix ``src_trace_`` are globally applied in the sc src_trace_config_from_toml ~~~~~~~~~~~~~~~~~~~~~~~~~~ -This configuration takes the (relative) path to a `toml file `__ which contains some or all of the ``CodeLinks`` configuration (configuration in the toml will override that in the :file:`conf.py`). +This configuration takes the (relative) path to a `toml file `__ +which contains some or all of the ``CodeLinks`` configuration +(configuration in the toml will override that in the :file:`conf.py`). .. code-block:: python @@ -26,11 +30,6 @@ This configuration takes the (relative) path to a `toml file `_ Configuration in the toml can contain any of the following options, under a ``[src_trace]`` section, but with the ``src_trace_`` prefix removed. -For example: - -.. literalinclude:: ./../../src_trace.toml - :language: toml - .. caution:: Any configuration specifying relative paths in the toml file will be resolved relatively to the directory containing the :file:`conf.py` file. .. _`src_trace_set_local_url`: @@ -298,6 +297,8 @@ Default: **True** .. attention:: The option currently do NOT support nested .gitignore +.. _oneline_comment_style: + oneline_comment_style ~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst new file mode 100644 index 0000000..c8d3dbe --- /dev/null +++ b/docs/source/components/oneline.rst @@ -0,0 +1,83 @@ +.. _oneline: + +One Line Comment Style +====================== + +Many users raised the concerns about the complication of defining Sphinx-Needs with RST in source code. +Therefore, ``CodeLinks`` provides a customizable one-line comment style pattern to define ``Sphinx-Needs`` +in order to simplify the efforts to create a need in source code. + +`Here `_ is the default one-line comment style. + +Start and End sequences +----------------------- + +To have better understanding of its the syntax of one-line comment, we will break it down to the following: + +**start_sequence** defines the characters where the one-line comment starts. +**end_sequence** defines the characters where the one-line comment ends. + +The text between **start_sequence** and **end_sequence** are fields of ``Sphinx-Needs`` + +field_split_char +---------------- + +There are always multiple fields for a need. Therefore, + +**field_split_char** defines the character to split the text into multiple ``pieces/fields``. + +needs_fields +------------ + +Each fields in a need may have different data types. +It could be a string if it is a field for ``id`` or ``title``. On the other hand, +it could be a list of string as well, if the field requires to have a list of string to represent ``links`` + +It's where **needs_fields** comes in. + +**needs_fields** contains the fields that is required for needs: + +Each field defines its: + +- name +- data type (Optional) +- default value (Optional) + +DataType +~~~~~~~~ + +By default, a field has the datatype of ``str``. + +For example, if the field definition is as follows: + +.. code-block:: python + + { + "name": "title + } + +It's equivalent to: + +.. code-block:: python + + { + "name": "title", + "type": "str" + } + +If the field is expected to have a list of strings, it shall be defined as the following: + +.. code-block:: python + + { + "name": "links", + "type": "list[str]" + } + +Default +~~~~~~~ + + + + +The ``order of needs_fields`` is important because it determines ``the position of the field`` in the one-line comment. diff --git a/docs/source/development/roadmap.rst b/docs/source/development/roadmap.rst index 1d11478..9e526e4 100644 --- a/docs/source/development/roadmap.rst +++ b/docs/source/development/roadmap.rst @@ -11,9 +11,23 @@ The other comment styles for different programming languages are planed, such as - Python - Rust +- YAML +- SyML Nested .gitignore ----------------- ``CodeLinks`` respects ``.gitignore`` file, but if the .gitignore files are nested, it's not supported. Respecting nested ``.gitignore`` in the context of the git repositories is planned. + +Flexible way to define Sphinx-Needs in source code +-------------------------------------------------- + +The only way to define ``Sphinx-Needs`` is through ``one-line comment style``. +Raw RST text and multi-lines comments style are planned to support + +Export needs.json +----------------- + +To facilitate CI workflow and enhance the portability of Sphinx-Needs defined in source code, +we plan to have the feature to export the needs defined in source code to a JSON file. From f5a60b499e42b5a4a309147ab0eedf569c89c320 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 9 Jul 2025 16:04:22 +0200 Subject: [PATCH 32/54] update oneline docs --- docs/source/components/oneline.rst | 126 +++++++++++++++++++++++++---- docs/source/index.rst | 1 + 2 files changed, 111 insertions(+), 16 deletions(-) diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst index c8d3dbe..f1e7d24 100644 --- a/docs/source/components/oneline.rst +++ b/docs/source/components/oneline.rst @@ -7,7 +7,7 @@ Many users raised the concerns about the complication of defining Sphinx-Needs w Therefore, ``CodeLinks`` provides a customizable one-line comment style pattern to define ``Sphinx-Needs`` in order to simplify the efforts to create a need in source code. -`Here `_ is the default one-line comment style. +:ref:`Here ` is the default one-line comment style. Start and End sequences ----------------------- @@ -37,12 +37,16 @@ It's where **needs_fields** comes in. **needs_fields** contains the fields that is required for needs: -Each field defines its: +Each need field defines its: - name - data type (Optional) - default value (Optional) +The examples in the following sections use :ref:`the default ` to +explain the syntax of the one-line comment. + + DataType ~~~~~~~~ @@ -52,32 +56,122 @@ For example, if the field definition is as follows: .. code-block:: python - { - "name": "title - } + { + "name": "title + } It's equivalent to: .. code-block:: python - { - "name": "title", - "type": "str" - } + { + "name": "title", + "type": "str" + } If the field is expected to have a list of strings, it shall be defined as the following: .. code-block:: python - { - "name": "links", - "type": "list[str]" - } + { + "name": "links", + "type": "list[str]" + } + +When the field has data type as ``list[str]``, + +- ``[`` and ``]`` shall be used to wrap the multiple strings +- ``,`` shall be used as the separator. + +For example, with the following **needs_fields** configuration: + +.. code-block:: python + + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + +the online line comment shall be defined as the following + +.. tabs:: + + .. code-tab:: c + + // @ title, id_123, implementation, [link1, link2] + + .. code-tab:: rst + + .. implementation:: title + :id: id_123 + :links: link1, link2 + + + +Default value +~~~~~~~~~~~~~ + +The value mapped to the key ``default`` in a need field definition is the default value of a need field, +when it is not given in the need definition. + +For example, with the following needs_fields definition, + +.. code-block:: python + + needs_fields = [ + { + "name": "title" + }, + { + "name": "type", + "default": "implementation" + }, + ] + +the following need definition in source code is equivalent to RST shown below: + +.. tabs:: + + .. code-tab:: c + + // @ title here and default is used for type + + .. code-tab:: rst + + .. implementation:: title here and default is used for type + +Positional Fields +~~~~~~~~~~~~~~~~~ + +All of the fields defined in ``needs_fields`` are positional fields. +It means the ``order of needs_fields`` determines ``the position of the field`` in the one-line comment. + +For example, with the following **needs_fields** definition, + +.. code-block:: python + + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + +field ``title`` is the first element is the list, so the string of the title must be +the first field in the one-line comment + +.. tabs:: -Default -~~~~~~~ + .. code-tab:: c + // @ this is title, this is id, this_type, [link1, link2] + .. code-tab:: rst + .. this_type:: this is title + :id: this is id + :links: link1, link2 -The ``order of needs_fields`` is important because it determines ``the position of the field`` in the one-line comment. +.. note:: A field without default can NOT follow a field that has default set. diff --git a/docs/source/index.rst b/docs/source/index.rst index 0e8d336..bd9be42 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -61,6 +61,7 @@ Contents :caption: Components components/configuration + components/oneline components/cli .. toctree:: From fe6a476fbd2e3de5a7735c6e88240478b1d1f390 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 9 Jul 2025 16:06:58 +0200 Subject: [PATCH 33/54] updated docs --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index bd9be42..de45179 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,7 +16,7 @@ ``Sphinx-CodeLinks`` is designed for Engineering-as-Code workflow to facilitate ALM. - It enables users to defined ``Sphinx-Needs`` within source code and automatically extract them + It enables users to defined ``Sphinx-Needs`` within source code in one-line and automatically extract them into the documentation during the Sphinx build process. .. grid:: 1 1 2 2 From 67c2ed7eaeece1804751ce11c1049f9a008d39fe Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Wed, 9 Jul 2025 16:17:53 +0200 Subject: [PATCH 34/54] added caution --- docs/source/components/configuration.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 1dc18aa..51310ca 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -356,4 +356,6 @@ it is equivalent to the following RST :id: IMPL_4 :links: [SPEC_1, SPEC_2] +.. caution:: **type** and **title** must be configured in **needs_fields** as they are mandatory for Sphinx-Needs + More uses cases can be found in `tests `__ From a7600aba12b71fc32e52489e2fa8579b16d5efd1 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 11:46:25 +0200 Subject: [PATCH 35/54] enhance online docs --- docs/source/components/oneline.rst | 65 +++++++++++++++++++++++++----- tests/test_virtual_docs.py | 11 +++++ 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst index f1e7d24..1aeb261 100644 --- a/docs/source/components/oneline.rst +++ b/docs/source/components/oneline.rst @@ -80,11 +80,13 @@ If the field is expected to have a list of strings, it shall be defined as the f When the field has data type as ``list[str]``, -- ``[`` and ``]`` shall be used to wrap the multiple strings +- the strings must be given within ``[`` and ``]`` brackets - ``,`` shall be used as the separator. For example, with the following **needs_fields** configuration: +.. _fields_config: + .. code-block:: python needs_fields=[ @@ -148,16 +150,8 @@ Positional Fields All of the fields defined in ``needs_fields`` are positional fields. It means the ``order of needs_fields`` determines ``the position of the field`` in the one-line comment. -For example, with the following **needs_fields** definition, - -.. code-block:: python - needs_fields=[ - {"name": "title"}, - {"name": "id"}, - {"name": "type", "default": "impl"}, - {"name": "links", "type": "list[str]", "default": []}, - ], +For example, with the mentioned :ref:`needs_fields definition ` field ``title`` is the first element is the list, so the string of the title must be the first field in the one-line comment @@ -175,3 +169,54 @@ the first field in the one-line comment :links: link1, link2 .. note:: A field without default can NOT follow a field that has default set. + +Escaping Characters +~~~~~~~~~~~~~~~~~~~~ + +If the value of the field contains the characters which is ``field_split_char`` or angular brackets ``[`` and ``]``, + +leading character ``\`` must be used to escape them. + +For example, with the mentioned :ref:`needs_fields definition `, +``,`` is escaped with ``\`` and is not considered as a separator. + +.. tabs:: + + .. code-tab:: c + + // @ title\, 3, IMPL_3 , impl, [] + + .. code-tab:: rst + + .. impl:: title, 3 + :id: IMPL_3 + +The other example, the angular brackets ``[`` and ``]`` and comma are escaped + +.. tabs:: + + .. code-tab:: c + + // @ title 3, IMPL_3 , impl, [\[SPEC\,_1\]] + + .. code-tab:: rst + + .. impl:: title 3 + :id: IMPL_3 + :links: [SPEC,_1] + +To have backwards slash ``\`` as a literal in the value, use ``\\`` as shown the following: + +.. tabs:: + + .. code-tab:: c + + // @ title\\ 3, IMPL_3 , impl, [\[SPEC\,_1\]] + + .. code-tab:: rst + + .. impl:: title\ 3 + :id: IMPL_3 + :links: [SPEC,_1] + +.. caution:: Field values can never have any newline chars ``\r`` ``\n`` diff --git a/tests/test_virtual_docs.py b/tests/test_virtual_docs.py index 42dc8bf..6bb7d32 100644 --- a/tests/test_virtual_docs.py +++ b/tests/test_virtual_docs.py @@ -336,6 +336,17 @@ def test_oneline_schema_validator_positive(oneline_config): "priority": "low", }, ), + ( + "[[IMPL_13, title\\ 13, impl, [\[SPEC\,_1\]], open]]", + { + "id": "IMPL_13", + "title": "title\ 13", + "type": "impl", + "links": ["[SPEC,_1]"], + "status": "open", + "priority": "low", + }, + ), ], ) def test_oneline_parser_custom_config_positive(oneline: str, result): From 1eba83116f27e3b5d08eae749e191bc895fb4994 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 11:46:38 +0200 Subject: [PATCH 36/54] added release action --- .github/workflows/gh_pages.yml | 55 ++++++++++++++++++++++++++++++++++ .github/workflows/release.yaml | 21 +++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 .github/workflows/gh_pages.yml create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/gh_pages.yml b/.github/workflows/gh_pages.yml new file mode 100644 index 0000000..1d8401c --- /dev/null +++ b/.github/workflows/gh_pages.yml @@ -0,0 +1,55 @@ +# Workflow for building and deploying the Sphinx site to GitHub Pages +# +name: Deploy docs to GH Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + name: Build + runs-on: [self-hosted, linux, x64] + steps: + - uses: actions/checkout@v4 + - name: Setup Pages + id: pages + uses: actions/configure-pages@v5 + - name: Install graphviz + run: sudo apt-get --yes install graphviz + - uses: eifinger/setup-rye@v4 + - run: rye sync + - name: Run documentation build + run: rye run docs + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/_build/html + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: [self-hosted, linux, x64] + name: Deploy + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..ec63111 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,21 @@ +name: Release +on: + push: + tags: + - '[0-9].[0-9]+.[0-9]+' + +jobs: + publish: + name: Publish to PyPi + # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + runs-on: [self-hosted, linux, x64] + steps: + - name: Checkout source + uses: actions/checkout@v4 + - name: install flit + run: pip install flit~=3.4 + - name: Build and publish + run: flit publish + env: + FLIT_USERNAME: __token__ + FLIT_PASSWORD: ${{ secrets.PYPI }} From ed0b2d05f704addfa89eaf73f7d8c12bdf4b06e1 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 13:13:46 +0200 Subject: [PATCH 37/54] test release action --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ec63111..522e517 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,8 +1,8 @@ name: Release on: push: - tags: - - '[0-9].[0-9]+.[0-9]+' + # tags: + # - '[0-9].[0-9]+.[0-9]+' jobs: publish: From 1f99f6269052dd1ecf8315cdda3c80b2a36912d1 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 13:21:42 +0200 Subject: [PATCH 38/54] test github_page --- .github/workflows/gh_pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh_pages.yml b/.github/workflows/gh_pages.yml index 1d8401c..6dc69f3 100644 --- a/.github/workflows/gh_pages.yml +++ b/.github/workflows/gh_pages.yml @@ -5,7 +5,7 @@ name: Deploy docs to GH Pages on: # Runs on pushes targeting the default branch push: - branches: ["main"] + # branches: ["main"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From da93659ee683c72294bddea499e7016a04646cb5 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 13:22:15 +0200 Subject: [PATCH 39/54] update release action --- .github/workflows/release.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 522e517..d31f306 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,8 +12,6 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v4 - - name: install flit - run: pip install flit~=3.4 - name: Build and publish run: flit publish env: From 4eba349920ea61439c248f1a66ffea06aa261637 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 13:25:22 +0200 Subject: [PATCH 40/54] use rye build --- .github/workflows/release.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d31f306..2071bd6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -12,8 +12,11 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v4 - - name: Build and publish - run: flit publish - env: - FLIT_USERNAME: __token__ - FLIT_PASSWORD: ${{ secrets.PYPI }} + - uses: ./.github/actions/setup_rye + - name: Build package + run: rye build + - name: Publish package + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From a916a490390838aff5495079129220087759def2 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 13:31:05 +0200 Subject: [PATCH 41/54] remove test gh page --- .github/workflows/gh_pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh_pages.yml b/.github/workflows/gh_pages.yml index 6dc69f3..1d8401c 100644 --- a/.github/workflows/gh_pages.yml +++ b/.github/workflows/gh_pages.yml @@ -5,7 +5,7 @@ name: Deploy docs to GH Pages on: # Runs on pushes targeting the default branch push: - # branches: ["main"] + branches: ["main"] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From baaf83c7b682a3368996090ada49cd617de4f7b3 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 13:47:59 +0200 Subject: [PATCH 42/54] specified package location --- .github/workflows/release.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2071bd6..dcaa60d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,6 +9,8 @@ jobs: name: Publish to PyPi # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') runs-on: [self-hosted, linux, x64] + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: - name: Checkout source uses: actions/checkout@v4 @@ -16,7 +18,6 @@ jobs: - name: Build package run: rye build - name: Publish package - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} + packages-dir: ./dist/ From 95e1bdda2c180f5a92ae10189d23250678abad86 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 13:56:59 +0200 Subject: [PATCH 43/54] updated permission --- .github/workflows/release.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dcaa60d..4d06667 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,13 +4,16 @@ on: # tags: # - '[0-9].[0-9]+.[0-9]+' +permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + contents: read + jobs: publish: name: Publish to PyPi # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') runs-on: [self-hosted, linux, x64] - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: - name: Checkout source uses: actions/checkout@v4 From 2e7dd458b2c7fc986f2c0168f49624f2dca01964 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 14:06:28 +0200 Subject: [PATCH 44/54] updated package-dir --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4d06667..b8fe8e8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,4 +23,4 @@ jobs: - name: Publish package uses: pypa/gh-action-pypi-publish@release/v1 with: - packages-dir: ./dist/ + packages-dir: sphinx-codelinks/dist/ From a4ed9fb090e2c4698533f7ef41160d5db18066a6 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 14:10:19 +0200 Subject: [PATCH 45/54] updated --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b8fe8e8..6c557e5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,4 +23,4 @@ jobs: - name: Publish package uses: pypa/gh-action-pypi-publish@release/v1 with: - packages-dir: sphinx-codelinks/dist/ + packages-dir: ${{ github.workspace }}/dist/ From b304b51648c5adafe8dfb69b220ed1c915bf32e5 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 14:27:41 +0200 Subject: [PATCH 46/54] updated --- .github/workflows/release.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6c557e5..78f38c8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -22,5 +22,3 @@ jobs: run: rye build - name: Publish package uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: ${{ github.workspace }}/dist/ From 5ab9d395684e311cc8ea2d4d43d3fddd8b1a7db9 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 14:49:24 +0200 Subject: [PATCH 47/54] updated --- .github/workflows/release.yaml | 41 +++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 78f38c8..c24c9d5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,16 +9,35 @@ permissions: contents: read jobs: - publish: - name: Publish to PyPi - # if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') - runs-on: [self-hosted, linux, x64] + build: + name: Build distribution 📦 + runs-on: ubuntu-latest steps: - - name: Checkout source - uses: actions/checkout@v4 - - uses: ./.github/actions/setup_rye - - name: Build package - run: rye build - - name: Publish package - uses: pypa/gh-action-pypi-publish@release/v1 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + needs: + - build + runs-on: ubuntu-latest + # environment: + # name: pypi + # url: https://pypi.org/p/ # Replace with your PyPI project name + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 From a3a5d7cf882d8078beb3bc3994e6b9236f4e1bbe Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 14:51:54 +0200 Subject: [PATCH 48/54] updated --- .github/workflows/release.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c24c9d5..9434cae 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -21,6 +21,19 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ publish-to-pypi: name: >- From 54fa062da3f1aa6a35f5c644035a6fc5f1459f71 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 10 Jul 2025 15:22:10 +0200 Subject: [PATCH 49/54] Reformatted docs --- docs/source/basics/installation.rst | 10 +- docs/source/basics/quickstart.rst | 2 - docs/source/components/cli.rst | 12 +-- docs/source/components/configuration.rst | 28 +++--- docs/source/components/oneline.rst | 116 +++++++++++------------ docs/source/development/contributing.rst | 3 +- docs/source/index.rst | 3 - 7 files changed, 82 insertions(+), 92 deletions(-) diff --git a/docs/source/basics/installation.rst b/docs/source/basics/installation.rst index 0f3fcc9..3aae289 100644 --- a/docs/source/basics/installation.rst +++ b/docs/source/basics/installation.rst @@ -4,7 +4,7 @@ Installation ============ Using Pip ----------- +--------- .. code-block:: bash @@ -17,7 +17,7 @@ For activation, please add ``sphinx_needs`` and ``sphinx-codelinks`` to the proj .. code-block:: python - extensions = [ - 'sphinx_needs', - 'sphinx_codelinks' - ] + extensions = [ + 'sphinx_needs', + 'sphinx_codelinks' + ] diff --git a/docs/source/basics/quickstart.rst b/docs/source/basics/quickstart.rst index 971fc2d..810b6bf 100644 --- a/docs/source/basics/quickstart.rst +++ b/docs/source/basics/quickstart.rst @@ -9,7 +9,6 @@ Quick Start :project: project_config :file: example.cpp - or .. code-block:: rst @@ -18,7 +17,6 @@ or :project: project_config :directory: ./example - ``src-trace`` directive has the following options: * **project**: the project config specified in ``conf.py`` or ``toml`` to be used for source tracing. diff --git a/docs/source/components/cli.rst b/docs/source/components/cli.rst index 52de229..e838d9b 100644 --- a/docs/source/components/cli.rst +++ b/docs/source/components/cli.rst @@ -7,9 +7,9 @@ and for local development. It features help pages. add ``-h`` or ``--help`` to any command to see the available options. .. typer:: sphinx_codelinks.cmd.app - :prog: codelinks - :width: 85 - :preferred: svg - :theme: monokai - :show-nested: - :make-sections: + :prog: codelinks + :width: 85 + :preferred: svg + :theme: monokai + :show-nested: + :make-sections: diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 51310ca..465c0be 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -41,7 +41,7 @@ Set this option to ``False``, if the local link between a need to the local sour Default: **True** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -61,7 +61,7 @@ Set the desired custom field name for the local link to the source code. Default: **local-url** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -84,7 +84,7 @@ The remote means where the source code is hosted such as GitHub. Default: **True** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -104,7 +104,7 @@ Set the desired custom field name for the remote link to the source code. Default: **remote-url** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -126,7 +126,7 @@ src_trace_projects This option contains multiple sets of project-specific options. The project name is defined as the key in a dictionary and its corresponding value is a dictionary containing the options specific to that project. -.. tabs:: +.. tabs:: .. code-tab:: python @@ -149,7 +149,7 @@ Default: **cpp** .. note:: Currently, only C/C++ is supported -.. tabs:: +.. tabs:: .. code-tab:: python @@ -171,7 +171,7 @@ The relative path based on ``conf.py`` file (NOT SURE) to the source code's root Default: **./** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -194,7 +194,7 @@ The pattern to access the source code to the remote repositories such as GitHub. Default: **Not set** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -234,7 +234,7 @@ The option is a list of glob patterns to exclude the files which are not require Default: **[]** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -256,7 +256,7 @@ The option is a list of glob patterns to include the files which are required to Default: **[]** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -281,7 +281,7 @@ The option to respect .gitignore :file Default: **True** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -297,7 +297,7 @@ Default: **True** .. attention:: The option currently do NOT support nested .gitignore -.. _oneline_comment_style: +.. _`oneline_comment_style`: oneline_comment_style ~~~~~~~~~~~~~~~~~~~~~ @@ -307,7 +307,7 @@ This option enables users to simply define a customized one-line-pattern comment Default: -.. tabs:: +.. tabs:: .. code-tab:: python @@ -344,7 +344,7 @@ Default: With the default, the following one-line comment will be extracted by ``CodeLinks`` and it is equivalent to the following RST -.. tabs:: +.. tabs:: .. code-tab:: c diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst index 1aeb261..027bd1c 100644 --- a/docs/source/components/oneline.rst +++ b/docs/source/components/oneline.rst @@ -46,7 +46,6 @@ Each need field defines its: The examples in the following sections use :ref:`the default ` to explain the syntax of the one-line comment. - DataType ~~~~~~~~ @@ -85,32 +84,30 @@ When the field has data type as ``list[str]``, For example, with the following **needs_fields** configuration: -.. _fields_config: +.. _`fields_config`: .. code-block:: python - needs_fields=[ - {"name": "title"}, - {"name": "id"}, - {"name": "type", "default": "impl"}, - {"name": "links", "type": "list[str]", "default": []}, - ], + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], the online line comment shall be defined as the following -.. tabs:: - - .. code-tab:: c +.. tabs:: - // @ title, id_123, implementation, [link1, link2] + .. code-tab:: c - .. code-tab:: rst - - .. implementation:: title - :id: id_123 - :links: link1, link2 + // @ title, id_123, implementation, [link1, link2] + .. code-tab:: rst + .. implementation:: title + :id: id_123 + :links: link1, link2 Default value ~~~~~~~~~~~~~ @@ -122,27 +119,27 @@ For example, with the following needs_fields definition, .. code-block:: python - needs_fields = [ - { - "name": "title" - }, - { - "name": "type", - "default": "implementation" - }, - ] + needs_fields = [ + { + "name": "title" + }, + { + "name": "type", + "default": "implementation" + }, + ] the following need definition in source code is equivalent to RST shown below: -.. tabs:: +.. tabs:: - .. code-tab:: c + .. code-tab:: c - // @ title here and default is used for type + // @ title here and default is used for type - .. code-tab:: rst + .. code-tab:: rst - .. implementation:: title here and default is used for type + .. implementation:: title here and default is used for type Positional Fields ~~~~~~~~~~~~~~~~~ @@ -150,28 +147,27 @@ Positional Fields All of the fields defined in ``needs_fields`` are positional fields. It means the ``order of needs_fields`` determines ``the position of the field`` in the one-line comment. - For example, with the mentioned :ref:`needs_fields definition ` field ``title`` is the first element is the list, so the string of the title must be the first field in the one-line comment -.. tabs:: +.. tabs:: - .. code-tab:: c + .. code-tab:: c - // @ this is title, this is id, this_type, [link1, link2] + // @ this is title, this is id, this_type, [link1, link2] - .. code-tab:: rst + .. code-tab:: rst - .. this_type:: this is title - :id: this is id - :links: link1, link2 + .. this_type:: this is title + :id: this is id + :links: link1, link2 .. note:: A field without default can NOT follow a field that has default set. Escaping Characters -~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~ If the value of the field contains the characters which is ``field_split_char`` or angular brackets ``[`` and ``]``, @@ -180,43 +176,43 @@ leading character ``\`` must be used to escape them. For example, with the mentioned :ref:`needs_fields definition `, ``,`` is escaped with ``\`` and is not considered as a separator. -.. tabs:: +.. tabs:: - .. code-tab:: c + .. code-tab:: c - // @ title\, 3, IMPL_3 , impl, [] + // @ title\, 3, IMPL_3 , impl, [] - .. code-tab:: rst + .. code-tab:: rst - .. impl:: title, 3 - :id: IMPL_3 + .. impl:: title, 3 + :id: IMPL_3 The other example, the angular brackets ``[`` and ``]`` and comma are escaped -.. tabs:: +.. tabs:: - .. code-tab:: c + .. code-tab:: c - // @ title 3, IMPL_3 , impl, [\[SPEC\,_1\]] + // @ title 3, IMPL_3 , impl, [\[SPEC\,_1\]] - .. code-tab:: rst + .. code-tab:: rst - .. impl:: title 3 - :id: IMPL_3 - :links: [SPEC,_1] + .. impl:: title 3 + :id: IMPL_3 + :links: [SPEC,_1] To have backwards slash ``\`` as a literal in the value, use ``\\`` as shown the following: -.. tabs:: +.. tabs:: - .. code-tab:: c + .. code-tab:: c - // @ title\\ 3, IMPL_3 , impl, [\[SPEC\,_1\]] + // @ title\\ 3, IMPL_3 , impl, [\[SPEC\,_1\]] - .. code-tab:: rst + .. code-tab:: rst - .. impl:: title\ 3 - :id: IMPL_3 - :links: [SPEC,_1] + .. impl:: title\ 3 + :id: IMPL_3 + :links: [SPEC,_1] .. caution:: Field values can never have any newline chars ``\r`` ``\n`` diff --git a/docs/source/development/contributing.rst b/docs/source/development/contributing.rst index b00b72f..2041bce 100644 --- a/docs/source/development/contributing.rst +++ b/docs/source/development/contributing.rst @@ -40,12 +40,11 @@ To run the formatting and linting, pre-commit is used: pre-commit install # to auto-run on every commit pre-commit run --all-files # to run manually - The CI also checks typing, use the following command locally to see if your code is well-typed .. code-block:: bash - rye run mypy:all + rye run mypy:all Build docs ---------- diff --git a/docs/source/index.rst b/docs/source/index.rst index de45179..695a7c4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,5 +1,3 @@ - - .. grid:: :class-row: sd-w-100 @@ -14,7 +12,6 @@ .. div:: sd-fs-5 sd-font-italic - ``Sphinx-CodeLinks`` is designed for Engineering-as-Code workflow to facilitate ALM. It enables users to defined ``Sphinx-Needs`` within source code in one-line and automatically extract them into the documentation during the Sphinx build process. From c0af6b2f9435f408076a9f3aefdd0ab0a4aa58be Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 10 Jul 2025 15:34:39 +0200 Subject: [PATCH 50/54] Rewrite some docs --- README.md | 44 +++++++++++++++++++----- docs/conf.py | 2 +- docs/source/basics/installation.rst | 2 +- docs/source/basics/quickstart.rst | 8 ++--- docs/source/components/configuration.rst | 10 +++--- docs/source/components/oneline.rst | 38 ++++++++++---------- docs/source/index.rst | 4 +-- 7 files changed, 68 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 3bd3505..d751126 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,36 @@ -* [ ] Move code -* [ ] Create doc project, be based on ubCode/ubTrace -* [ ] ubCode ubproject.toml -* [ ] CI for docs and tests -* [ ] DNS for codelinks.useblocks.com -* [ ] Deployment on pypi (see Sphinx-Needs) -* [ ] Repo rules (no main pushes / branch protection) -* [ ] Cleanup ubTrace (files, ci, rye commands) +# Sphinx CodeLinks + +A Sphinx extension for discovering, linking, and documenting source code across projects. + +## Features + +- **Source Discovery**: Automatically discover source files in your project +- **Virtual Documentation**: Generate documentation from code without modifying source files +- **Code Linking**: Create intelligent links between code elements +- **Sphinx Integration**: Seamless integration with existing Sphinx documentation + +## Quick Start + +```bash +pip install sphinx-codelinks +``` + +Add to your `conf.py`: +```python +extensions = ['sphinx_codelinks'] +``` + +## Documentation + +Full documentation: https://codelinks.useblocks.com + +## Components + +- **Source Discovery** ([`src/sphinx_codelinks/source_discovery`](src/sphinx_codelinks/source_discovery)): Code analysis and discovery +- **Virtual Docs** ([`src/sphinx_codelinks/virtual_docs`](src/sphinx_codelinks/virtual_docs)): Documentation generation +- **Sphinx Extension** ([`src/sphinx_codelinks/sphinx_extension`](src/sphinx_codelinks/sphinx_extension)): Sphinx integration +- **Command Line** ([`src/sphinx_codelinks/cmd.py`](src/sphinx_codelinks/cmd.py)): CLI interface + +## Development + +See [Development Guide](docs/source/development/) for contributing guidelines. diff --git a/docs/conf.py b/docs/conf.py index 87494b8..72ab3a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,7 @@ (Path(__file__).parent.parent / "pyproject.toml").read_text("utf8") )["project"] -project = "ubtrace" +project = _project_data['name'] author = _project_data["authors"][0]["name"] copyright = f"{datetime.now().year}, {author}" version = release = _project_data["version"] diff --git a/docs/source/basics/installation.rst b/docs/source/basics/installation.rst index 3aae289..2dba60d 100644 --- a/docs/source/basics/installation.rst +++ b/docs/source/basics/installation.rst @@ -13,7 +13,7 @@ Using Pip Activation ---------- -For activation, please add ``sphinx_needs`` and ``sphinx-codelinks`` to the projects's extension list of your **conf.py** file +For activation, please add ``sphinx_needs`` and ``sphinx_codelinks`` to the project's extension list in your **conf.py** file .. code-block:: python diff --git a/docs/source/basics/quickstart.rst b/docs/source/basics/quickstart.rst index 810b6bf..d21ca8e 100644 --- a/docs/source/basics/quickstart.rst +++ b/docs/source/basics/quickstart.rst @@ -19,14 +19,14 @@ or ``src-trace`` directive has the following options: -* **project**: the project config specified in ``conf.py`` or ``toml`` to be used for source tracing. +* **project**: the project config specified in ``conf.py`` or ``toml`` file to be used for source tracing. * **file**: the source file to be traced. * **directory**: the source files in the directory to be traced recursively. -Regarding the options **file** and **directory**: +Regarding the **file** and **directory** options: - they are optional and mutually exclusive. -- the given paths of them are relative to ``src_dir`` defined in the source tracing configuration +- the given paths are relative to ``src_dir`` defined in the source tracing configuration - if not given, the whole project will be examined. Example @@ -71,4 +71,4 @@ The needs defined in source code are extracted and rendered to: :project: dcdc :directory: ./discharge -To have more customized configuration of ``CodeLinks``, please refer to :ref:`configuration ` +To have a more customized configuration of ``CodeLinks``, please refer to :ref:`configuration `. diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 465c0be..53e9252 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -55,7 +55,7 @@ Default: **True** src_trace_set_local_field ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: This option is only optionally required, if :ref:`src_trace_set_local_url` is set to **True**. +.. note:: This option is only required if :ref:`src_trace_set_local_url` is set to **True**. Set the desired custom field name for the local link to the source code. @@ -98,7 +98,7 @@ Default: **True** src_trace_set_remote_field ~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: This option is only optionally required, if :ref:`src_trace_set_remote_url` is set to **True**. +.. note:: This option is only required if :ref:`src_trace_set_remote_url` is set to **True**. Set the desired custom field name for the remote link to the source code. @@ -167,7 +167,7 @@ Default: **cpp** src_dir ~~~~~~~ -The relative path based on ``conf.py`` file (NOT SURE) to the source code's root directory +The relative path from the ``conf.py`` file to the source code's root directory Default: **./** @@ -277,7 +277,7 @@ Default: **[]** gitignore ~~~~~~~~~ -The option to respect .gitignore :file +The option to respect the .gitignore file. Default: **True** @@ -295,7 +295,7 @@ Default: **True** [src_trace.projects.project_name] gitignore = false -.. attention:: The option currently do NOT support nested .gitignore +.. attention:: This option currently does NOT support nested .gitignore files .. _`oneline_comment_style`: diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst index 027bd1c..90f3806 100644 --- a/docs/source/components/oneline.rst +++ b/docs/source/components/oneline.rst @@ -3,9 +3,9 @@ One Line Comment Style ====================== -Many users raised the concerns about the complication of defining Sphinx-Needs with RST in source code. -Therefore, ``CodeLinks`` provides a customizable one-line comment style pattern to define ``Sphinx-Needs`` -in order to simplify the efforts to create a need in source code. +Many users have raised concerns about the complexity of defining Sphinx-Needs with RST in source code. +Therefore, ``CodeLinks`` provides a customizable one-line comment style pattern to define ``Sphinx-Needs`` +to simplify the effort required to create a need in source code. :ref:`Here ` is the default one-line comment style. @@ -17,25 +17,25 @@ To have better understanding of its the syntax of one-line comment, we will brea **start_sequence** defines the characters where the one-line comment starts. **end_sequence** defines the characters where the one-line comment ends. -The text between **start_sequence** and **end_sequence** are fields of ``Sphinx-Needs`` +The text between **start_sequence** and **end_sequence** contains the fields of ``Sphinx-Needs`` field_split_char ---------------- -There are always multiple fields for a need. Therefore, +Since there are always multiple fields for a need, **field_split_char** defines the character to split the text into multiple ``pieces/fields``. needs_fields ------------ -Each fields in a need may have different data types. +Each field in a need may have different data types. It could be a string if it is a field for ``id`` or ``title``. On the other hand, -it could be a list of string as well, if the field requires to have a list of string to represent ``links`` +it could be a list of strings as well, if the field requires a list of strings to represent ``links``. -It's where **needs_fields** comes in. +This is where **needs_fields** comes in. -**needs_fields** contains the fields that is required for needs: +**needs_fields** contains the fields that are required for needs: Each need field defines its: @@ -49,7 +49,7 @@ explain the syntax of the one-line comment. DataType ~~~~~~~~ -By default, a field has the datatype of ``str``. +By default, a field has the data type of ``str``. For example, if the field definition is as follows: @@ -77,7 +77,7 @@ If the field is expected to have a list of strings, it shall be defined as the f "type": "list[str]" } -When the field has data type as ``list[str]``, +When the field has data type ``list[str]``: - the strings must be given within ``[`` and ``]`` brackets - ``,`` shall be used as the separator. @@ -112,7 +112,7 @@ the online line comment shall be defined as the following Default value ~~~~~~~~~~~~~ -The value mapped to the key ``default`` in a need field definition is the default value of a need field, +The value mapped to the key ``default`` in a need field definition is the default value of a need field when it is not given in the need definition. For example, with the following needs_fields definition, @@ -145,7 +145,7 @@ Positional Fields ~~~~~~~~~~~~~~~~~ All of the fields defined in ``needs_fields`` are positional fields. -It means the ``order of needs_fields`` determines ``the position of the field`` in the one-line comment. +This means the ``order of needs_fields`` determines ``the position of the field`` in the one-line comment. For example, with the mentioned :ref:`needs_fields definition ` @@ -164,14 +164,14 @@ the first field in the one-line comment :id: this is id :links: link1, link2 -.. note:: A field without default can NOT follow a field that has default set. +.. note:: A field without a default value cannot follow a field that has a default value set. Escaping Characters ~~~~~~~~~~~~~~~~~~~ -If the value of the field contains the characters which is ``field_split_char`` or angular brackets ``[`` and ``]``, +If the value of the field contains characters that are ``field_split_char`` or angular brackets ``[`` and ``]``, -leading character ``\`` must be used to escape them. +a leading character ``\`` must be used to escape them. For example, with the mentioned :ref:`needs_fields definition `, ``,`` is escaped with ``\`` and is not considered as a separator. @@ -187,7 +187,7 @@ For example, with the mentioned :ref:`needs_fields definition `, .. impl:: title, 3 :id: IMPL_3 -The other example, the angular brackets ``[`` and ``]`` and comma are escaped +The other example shows the angular brackets ``[`` and ``]`` and comma being escaped: .. tabs:: @@ -201,7 +201,7 @@ The other example, the angular brackets ``[`` and ``]`` and comma are escaped :id: IMPL_3 :links: [SPEC,_1] -To have backwards slash ``\`` as a literal in the value, use ``\\`` as shown the following: +To have a backslash ``\`` as a literal in the value, use ``\\`` as shown in the following: .. tabs:: @@ -215,4 +215,4 @@ To have backwards slash ``\`` as a literal in the value, use ``\\`` as shown the :id: IMPL_3 :links: [SPEC,_1] -.. caution:: Field values can never have any newline chars ``\r`` ``\n`` +.. caution:: Field values can never contain any newline characters ``\r`` or ``\n``. diff --git a/docs/source/index.rst b/docs/source/index.rst index 695a7c4..41f213d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -12,8 +12,8 @@ .. div:: sd-fs-5 sd-font-italic - ``Sphinx-CodeLinks`` is designed for Engineering-as-Code workflow to facilitate ALM. - It enables users to defined ``Sphinx-Needs`` within source code in one-line and automatically extract them + ``Sphinx-CodeLinks`` is designed for Engineering-as-Code workflows to facilitate ALM. + It enables users to define ``Sphinx-Needs`` within source code in one line and automatically extract them into the documentation during the Sphinx build process. .. grid:: 1 1 2 2 From 6521dbb8b724ca7fc8e53343f852e3634a0da9ce Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 10 Jul 2025 15:41:47 +0200 Subject: [PATCH 51/54] Wording --- docs/source/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 41f213d..710e85b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -12,8 +12,8 @@ .. div:: sd-fs-5 sd-font-italic - ``Sphinx-CodeLinks`` is designed for Engineering-as-Code workflows to facilitate ALM. - It enables users to define ``Sphinx-Needs`` within source code in one line and automatically extract them + ``Sphinx-CodeLinks`` is designed for Engineering-as-Code workflows to facilitate Application Lifecycle Management (ALM). + It enables users to define ``Sphinx-Needs`` need items within source code using a single line and automatically extract them into the documentation during the Sphinx build process. .. grid:: 1 1 2 2 From 8ffee7a365d8c5b8c2155971e97066a82a93cef9 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 16:11:31 +0200 Subject: [PATCH 52/54] updated according to the review --- .github/workflows/ci.yml | 28 ------------------------ .github/workflows/gh_pages.yml | 2 -- .github/workflows/release.yaml | 7 ++---- .pre-commit-config.yaml | 2 +- README.md | 3 ++- conftest.py | 19 ---------------- docs/conf.py | 12 +--------- docs/source/basics/introduction.rst | 2 +- docs/source/components/configuration.rst | 28 ++++++++++++------------ docs/source/components/oneline.rst | 18 +++++++-------- docs/source/development/change_log.rst | 2 +- docs/source/development/contributing.rst | 2 +- docs/source/development/roadmap.rst | 8 +++---- pyproject.toml | 2 -- src/sphinx_codelinks/__init__.py | 2 +- tests/conftest.py | 2 ++ 16 files changed, 39 insertions(+), 100 deletions(-) delete mode 100644 conftest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef7dcdd..d1e0f8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,39 +62,12 @@ jobs: - uses: ./.github/actions/setup_rye - run: rye test -a - pytest-prod: - # pytest against packages installed as they would be in production - # i.e. as wheels that may contain obfuscated code - - name: Pytest prod (${{ matrix.os }}-${{ matrix.arch }}) - strategy: - fail-fast: false - matrix: - include: - - os: linux - arch: x64 - - os: linux - arch: arm64 - - os: windows - arch: x64 - - os: macos - arch: arm64 - - runs-on: [self-hosted, "${{ matrix.os }}", "${{ matrix.arch }}"] - - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/setup_rye - - run: rye run pytest:prod - docs: name: Documentation build runs-on: [self-hosted, linux, x64] steps: - uses: actions/checkout@v4 - - name: Install graphviz - run: sudo apt-get --yes install graphviz - uses: ./.github/actions/setup_rye - name: Run documentation build run: rye run docs @@ -109,7 +82,6 @@ jobs: - pre-commit - mypy - pytest - - pytest-prod - docs runs-on: [self-hosted, linux, x64] diff --git a/.github/workflows/gh_pages.yml b/.github/workflows/gh_pages.yml index 1d8401c..62ab44c 100644 --- a/.github/workflows/gh_pages.yml +++ b/.github/workflows/gh_pages.yml @@ -31,8 +31,6 @@ jobs: - name: Setup Pages id: pages uses: actions/configure-pages@v5 - - name: Install graphviz - run: sudo apt-get --yes install graphviz - uses: eifinger/setup-rye@v4 - run: rye sync - name: Run documentation build diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9434cae..d215458 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,8 +1,8 @@ name: Release on: push: - # tags: - # - '[0-9].[0-9]+.[0-9]+' + tags: + - '[0-9].[0-9]+.[0-9]+' permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing @@ -41,9 +41,6 @@ jobs: needs: - build runs-on: ubuntu-latest - # environment: - # name: pypi - # url: https://pypi.org/p/ # Replace with your PyPI project name permissions: id-token: write # IMPORTANT: mandatory for trusted publishing steps: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da4d0bb..25cd2aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: types: [file] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.12.2 hooks: - id: ruff-format name: python format diff --git a/README.md b/README.md index d751126..a275cc8 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,9 @@ pip install sphinx-codelinks ``` Add to your `conf.py`: + ```python -extensions = ['sphinx_codelinks'] +extensions = ['sphinx_needs', 'sphinx_codelinks'] ``` ## Documentation diff --git a/conftest.py b/conftest.py deleted file mode 100644 index c58dfaa..0000000 --- a/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Global pytest conftest.py. - -This is needed due to: - -pytest test discovery error for workspace: /home/marco/ub/ubtrace - Failed: Defining 'pytest_plugins' in a non-top-level conftest is no longer supported: -It affects the entire test suite instead of just below the conftest as expected. - /home/marco/ub/ubtrace/python/ubt_connect_core/tests/conftest.py -Please move it to a top level conftest file at the rootdir: - /home/marco/ub/ubtrace -For more information, visit: - https://docs.pytest.org/en/stable/deprecations.html#pytest-plugins-in-non-top-level-conftest-files - -See also the root README.md. -""" - -# Makes make_app avaialble -pytest_plugins = ("sphinx.testing.fixtures",) diff --git a/docs/conf.py b/docs/conf.py index 72ab3a3..eca06af 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,7 @@ (Path(__file__).parent.parent / "pyproject.toml").read_text("utf8") )["project"] -project = _project_data['name'] +project = _project_data["name"] author = _project_data["authors"][0]["name"] copyright = f"{datetime.now().year}, {author}" version = release = _project_data["version"] @@ -65,13 +65,3 @@ html_css_files = ["furo.css"] src_trace_config_from_toml = "./src_trace.toml" - -needs_types = [ - { - "directive": "impl", - "title": "Implementation", - "prefix": "IMPL_", - "color": "#DF744A", - "style": "node", - }, -] diff --git a/docs/source/basics/introduction.rst b/docs/source/basics/introduction.rst index e83f94f..9066e39 100644 --- a/docs/source/basics/introduction.rst +++ b/docs/source/basics/introduction.rst @@ -2,7 +2,7 @@ Introduction ============ ``CodeLinks`` is a sphinx extension that provides a directive ``src-trace`` -to trace the :external+needs:doc:`Sphinx-Needs ` defined in source files. +to trace the :external+needs:doc:`Sphinx-Needs ` need items defined in source files. Instead of putting RST syntax in the comment, the need definition in source code is simplified to one-liner only, so that users can just write their `customized one-line comment `_ to have the traceability diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 53e9252..d29bb56 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -41,7 +41,7 @@ Set this option to ``False``, if the local link between a need to the local sour Default: **True** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -61,7 +61,7 @@ Set the desired custom field name for the local link to the source code. Default: **local-url** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -84,7 +84,7 @@ The remote means where the source code is hosted such as GitHub. Default: **True** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -104,7 +104,7 @@ Set the desired custom field name for the remote link to the source code. Default: **remote-url** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -126,7 +126,7 @@ src_trace_projects This option contains multiple sets of project-specific options. The project name is defined as the key in a dictionary and its corresponding value is a dictionary containing the options specific to that project. -.. tabs:: +.. tabs:: .. code-tab:: python @@ -149,7 +149,7 @@ Default: **cpp** .. note:: Currently, only C/C++ is supported -.. tabs:: +.. tabs:: .. code-tab:: python @@ -171,7 +171,7 @@ The relative path from the ``conf.py`` file to the source code's root directory Default: **./** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -194,7 +194,7 @@ The pattern to access the source code to the remote repositories such as GitHub. Default: **Not set** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -234,7 +234,7 @@ The option is a list of glob patterns to exclude the files which are not require Default: **[]** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -256,7 +256,7 @@ The option is a list of glob patterns to include the files which are required to Default: **[]** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -281,7 +281,7 @@ The option to respect the .gitignore file. Default: **True** -.. tabs:: +.. tabs:: .. code-tab:: python @@ -303,11 +303,11 @@ oneline_comment_style ~~~~~~~~~~~~~~~~~~~~~ This option enables users to simply define a customized one-line-pattern comment to represent -``Sphinx-Needs`` instead of using RST. +``Sphinx-Needs`` need items instead of using RST. Default: -.. tabs:: +.. tabs:: .. code-tab:: python @@ -344,7 +344,7 @@ Default: With the default, the following one-line comment will be extracted by ``CodeLinks`` and it is equivalent to the following RST -.. tabs:: +.. tabs:: .. code-tab:: c diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst index 90f3806..1867d78 100644 --- a/docs/source/components/oneline.rst +++ b/docs/source/components/oneline.rst @@ -3,8 +3,8 @@ One Line Comment Style ====================== -Many users have raised concerns about the complexity of defining Sphinx-Needs with RST in source code. -Therefore, ``CodeLinks`` provides a customizable one-line comment style pattern to define ``Sphinx-Needs`` +Many users have raised concerns about the complexity of defining ``Sphinx-Needs`` need items with RST in source code. +Therefore, ``CodeLinks`` provides a customizable one-line comment style pattern to define ``a need items`` to simplify the effort required to create a need in source code. :ref:`Here ` is the default one-line comment style. @@ -17,7 +17,7 @@ To have better understanding of its the syntax of one-line comment, we will brea **start_sequence** defines the characters where the one-line comment starts. **end_sequence** defines the characters where the one-line comment ends. -The text between **start_sequence** and **end_sequence** contains the fields of ``Sphinx-Needs`` +The text between **start_sequence** and **end_sequence** contains the fields of ``need items`` field_split_char ---------------- @@ -97,7 +97,7 @@ For example, with the following **needs_fields** configuration: the online line comment shall be defined as the following -.. tabs:: +.. tabs:: .. code-tab:: c @@ -131,7 +131,7 @@ For example, with the following needs_fields definition, the following need definition in source code is equivalent to RST shown below: -.. tabs:: +.. tabs:: .. code-tab:: c @@ -152,7 +152,7 @@ For example, with the mentioned :ref:`needs_fields definition ` field ``title`` is the first element is the list, so the string of the title must be the first field in the one-line comment -.. tabs:: +.. tabs:: .. code-tab:: c @@ -176,7 +176,7 @@ a leading character ``\`` must be used to escape them. For example, with the mentioned :ref:`needs_fields definition `, ``,`` is escaped with ``\`` and is not considered as a separator. -.. tabs:: +.. tabs:: .. code-tab:: c @@ -189,7 +189,7 @@ For example, with the mentioned :ref:`needs_fields definition `, The other example shows the angular brackets ``[`` and ``]`` and comma being escaped: -.. tabs:: +.. tabs:: .. code-tab:: c @@ -203,7 +203,7 @@ The other example shows the angular brackets ``[`` and ``]`` and comma being esc To have a backslash ``\`` as a literal in the value, use ``\\`` as shown in the following: -.. tabs:: +.. tabs:: .. code-tab:: c diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst index ce4f10b..c1ff73d 100644 --- a/docs/source/development/change_log.rst +++ b/docs/source/development/change_log.rst @@ -14,4 +14,4 @@ This version features: - Sphinx Directive ``src-trace`` - Virtual Docs and Source Discovery CLI -- One-line comment to define ``Sphinx-Needs`` +- One-line comment to define a ``Sphinx-Needs`` need item diff --git a/docs/source/development/contributing.rst b/docs/source/development/contributing.rst index 2041bce..03b3128 100644 --- a/docs/source/development/contributing.rst +++ b/docs/source/development/contributing.rst @@ -62,7 +62,7 @@ To run test cases locally: .. code-block:: bash - rye run pytest:prod + rye test -a Note some tests use `syrupy `__ to perform snapshot testing. These snapshots can be updated by running: diff --git a/docs/source/development/roadmap.rst b/docs/source/development/roadmap.rst index 9e526e4..b21529a 100644 --- a/docs/source/development/roadmap.rst +++ b/docs/source/development/roadmap.rst @@ -20,14 +20,14 @@ Nested .gitignore ``CodeLinks`` respects ``.gitignore`` file, but if the .gitignore files are nested, it's not supported. Respecting nested ``.gitignore`` in the context of the git repositories is planned. -Flexible way to define Sphinx-Needs in source code --------------------------------------------------- +Flexible way to define Sphinx-Needs need items in source code +------------------------------------------------------------- -The only way to define ``Sphinx-Needs`` is through ``one-line comment style``. +The only way to define ``Sphinx-Needs`` need items is through ``one-line comment style``. Raw RST text and multi-lines comments style are planned to support Export needs.json ----------------- -To facilitate CI workflow and enhance the portability of Sphinx-Needs defined in source code, +To facilitate CI workflow and enhance the portability of ``need items`` defined in source code, we plan to have the feature to export the needs defined in source code to a JSON file. diff --git a/pyproject.toml b/pyproject.toml index 5cfbec3..2c23c04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,8 +67,6 @@ codelinks = "sphinx_codelinks.cmd:app" "docs:rm" = "rm -rf docs/_build/html" "docs" = "sphinx-build -nW --keep-going -b html -T -c docs docs/source docs/_build/html" "docs:clean" = { chain = ["docs:rm", "docs"] } -# pytest prod -"pytest:prod" = { cmd = "uv run --with-requirements requirements.lock --no-editable --refresh pytest tests/" } [tool.ruff.lint] extend-select = [ diff --git a/src/sphinx_codelinks/__init__.py b/src/sphinx_codelinks/__init__.py index fa49be7..00b724d 100644 --- a/src/sphinx_codelinks/__init__.py +++ b/src/sphinx_codelinks/__init__.py @@ -1,4 +1,4 @@ -"""ubTrace source code analyzer""" +"""CodeLinks source code analyzer""" from sphinx_codelinks.sphinx_extension.source_tracing import setup diff --git a/tests/conftest.py b/tests/conftest.py index 09be6d6..dd9dd67 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,8 @@ import pytest from syrupy.extensions.single_file import SingleFileSnapshotExtension, WriteMode +pytest_plugins = "sphinx.testing.fixtures" + TEST_DIR = Path(__file__).parent SRC_TRACE_TOML = TEST_DIR / "data" / "sphinx" / "src_trace.toml" BASIC_VDOC_TOML = TEST_DIR / "data" / "oneline_comment_basic" / "vdoc_config.toml" From a68da4583a922c589c88d9e9cf0e8922d1344750 Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 16:48:39 +0200 Subject: [PATCH 53/54] make CI happy --- .pre-commit-config.yaml | 4 ---- pyproject.toml | 6 +++++- .../sphinx_extension/directives/src_trace.py | 4 ++-- src/sphinx_codelinks/virtual_docs/ubt_models.py | 6 ++++++ src/sphinx_codelinks/virtual_docs/virtual_docs.py | 6 +++--- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 25cd2aa..4932b6f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,10 +4,6 @@ repos: hooks: - id: end-of-file-fixer - id: trailing-whitespace - - id: pretty-format-json - args: [--autofix, --no-sort-keys] - files: (package\.json|tsconfig\.json)$ - types: [file] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.2 diff --git a/pyproject.toml b/pyproject.toml index 2c23c04..09d868e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,11 @@ force-sort-within-sections = true "S603", # subprocess-without-shell-equals-true - build scripts ] "src/sphinx_codelinks/sphinx_extension/debug.py" = [ - "T201", # print - used for output + "T201", # print - used for output + "UP047", # on-pep695-generic-function - it's generic +] +"src/sphinx_codelinks/cmd.py" = [ + "PLC0415", # import on top - only import relevant modules by use cases ] [tool.mypy] diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py index bec378b..1a5b2db 100644 --- a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -49,7 +49,7 @@ def generate_str_link_name( def get_git_commit_id(src_dir: Path) -> str: try: commit_id = ( - subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=src_dir) # noqa: S607, S603 + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=src_dir) # noqa: S607 .decode("utf-8") .strip() ) @@ -63,7 +63,7 @@ def get_git_commit_id(src_dir: Path) -> str: def get_git_root(cwd: Path = Path()) -> Path | None: try: # Run the git command to get the root directory - git_root = subprocess.check_output( # noqa: S603 + git_root = subprocess.check_output( ["git", "rev-parse", "--show-toplevel"], # noqa: S607 cwd=cwd, text=True, # Ensures the output is a string diff --git a/src/sphinx_codelinks/virtual_docs/ubt_models.py b/src/sphinx_codelinks/virtual_docs/ubt_models.py index d18cf3e..9bd2c89 100644 --- a/src/sphinx_codelinks/virtual_docs/ubt_models.py +++ b/src/sphinx_codelinks/virtual_docs/ubt_models.py @@ -36,6 +36,9 @@ def __eq__(self, value): return self.__dict__ == value.__dict__ return False + def __hash__(self) -> int: + return hash(self.__dict__) + def to_dict(self) -> dict[str, dict[str, str | list[str]] | str | int]: return { "text": self.text, @@ -67,6 +70,9 @@ def __eq__(self, value): return self.__dict__ == value.__dict__ return False + def __hash__(self) -> int: + return hash(self.__dict__) + def add_comment(self, comment: UBTComment) -> None: self.comments.append(comment) diff --git a/src/sphinx_codelinks/virtual_docs/virtual_docs.py b/src/sphinx_codelinks/virtual_docs/virtual_docs.py index c9d1b51..62bb299 100644 --- a/src/sphinx_codelinks/virtual_docs/virtual_docs.py +++ b/src/sphinx_codelinks/virtual_docs/virtual_docs.py @@ -4,6 +4,9 @@ import os from pathlib import Path +from comment_parser.parsers.c_parser import ( # type: ignore[import-untyped] + extract_comments, +) from comment_parser.parsers.common import Comment # type: ignore[import-untyped] from sphinx_codelinks.virtual_docs.config import ( @@ -63,9 +66,6 @@ def collect(self) -> None: raise Exception( f"Unsupported comment type: {self.comment_type}. Supported types are: {SUPPORTED_COMMENT_TYPES}." ) - from comment_parser.parsers.c_parser import ( # type: ignore[import-untyped] - extract_comments, - ) virtual_docs = [] self.load_virtual_docs() From 65663b5d7d7b7e842aa3bb50465c5e2321aa757c Mon Sep 17 00:00:00 2001 From: "jui-wen.chen" Date: Thu, 10 Jul 2025 17:48:21 +0200 Subject: [PATCH 54/54] confirm relative path --- docs/source/components/configuration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index d29bb56..ded60a3 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -30,7 +30,7 @@ which contains some or all of the ``CodeLinks`` configuration Configuration in the toml can contain any of the following options, under a ``[src_trace]`` section, but with the ``src_trace_`` prefix removed. -.. caution:: Any configuration specifying relative paths in the toml file will be resolved relatively to the directory containing the :file:`conf.py` file. +.. caution:: Any configuration specifying relative paths in the toml file will be resolved relatively to the directory containing the ``toml`` file. .. _`src_trace_set_local_url`: @@ -167,7 +167,7 @@ Default: **cpp** src_dir ~~~~~~~ -The relative path from the ``conf.py`` file to the source code's root directory +The relative path from the ``conf.py`` or ``.toml`` file to the source code's root directory Default: **./**