diff --git a/.github/actions/setup_rye/action.yml b/.github/actions/setup_rye/action.yml new file mode 100644 index 0000000..e370827 --- /dev/null +++ b/.github/actions/setup_rye/action.yml @@ -0,0 +1,31 @@ +name: Set up rye +runs: + using: 'composite' + steps: + # rye uses uv under the hood, so we need to set the cache directory correctly, based on the OS + - name: Set UV_CACHE_DIR for Linux + if: runner.os == 'Linux' + run: | + echo "UV_CACHE_DIR=/home/runner/.cache/uv" >> $GITHUB_ENV + shell: bash + - name: Set MATURIN_PEP517_ARGS for Linux + if: runner.os == 'Linux' + # make sure we always use zig, to get manylinux2014 compatible rust binaries + run: | + echo "MATURIN_PEP517_ARGS=--zig" >> $GITHUB_ENV + shell: bash + - name: Set UV_CACHE_DIR for MacOS + if: runner.os == 'macOS' + run: echo "UV_CACHE_DIR=/Users/gh-runner/Library/Caches/uv" >> $GITHUB_ENV + shell: bash + - name: Set UV_CACHE_DIR for Windows + if: runner.os == 'Windows' + run: echo "UV_CACHE_DIR=C:\\Users\\useblocks\\AppData\\Local\\uv-${{ runner.name }}" >> $env:GITHUB_ENV + shell: pwsh + # now install rye and sync the dependencies + - uses: eifinger/setup-rye@v4 + with: + version: "0.42.0" + enable-cache: false + - run: rye sync + shell: ${{ runner.os == 'Windows' && 'powershell' || 'bash' }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d1e0f8f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,93 @@ +name: ci + +on: + push: + branches: [main] + tags: + - "v[0-9]+.[0-9]+.[0-9]+*" + pull_request: + types: [closed, labeled, reopened, unlabeled, synchronize, opened] + +concurrency: + # For PRs, cancel in progress runs, if a new commit is pushed + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# These permissions are needed to interact with GitHub's OIDC Token endpoint. +permissions: + id-token: write + contents: read + +jobs: + pre-commit: + name: Pre-commit + runs-on: [self-hosted, linux, x64] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version-file: ".python-version" + - run: python -m pip install pre-commit pre-commit-uv + # - uses: pre-commit/action@v3.0.1 # note we don't use this, since it calls ations/cache, which actually takes longer than without it + - run: pre-commit run --all --show-diff-on-failure --color=always + + mypy: + name: MyPy + runs-on: [self-hosted, linux, x64] + + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup_rye + - run: rye run mypy:all + + pytest: + name: Pytest (${{ matrix.os }}-${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - os: linux + arch: x64 + - os: linux + arch: arm64 + - os: windows + arch: x64 + - os: macos + arch: arm64 + + runs-on: [self-hosted, "${{ matrix.os }}", "${{ matrix.arch }}"] + + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup_rye + - run: rye test -a + + docs: + name: Documentation build + runs-on: [self-hosted, linux, x64] + + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup_rye + - name: Run documentation build + run: rye run docs + + all_good: + # This job does nothing and is only used for the branch protection + # see https://github.com/marketplace/actions/alls-green#why + + if: ${{ !cancelled() }} + + needs: + - pre-commit + - mypy + - pytest + - docs + + runs-on: [self-hosted, linux, x64] + + steps: + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/gh_pages.yml b/.github/workflows/gh_pages.yml new file mode 100644 index 0000000..62ab44c --- /dev/null +++ b/.github/workflows/gh_pages.yml @@ -0,0 +1,53 @@ +# Workflow for building and deploying the Sphinx site to GitHub Pages +# +name: Deploy docs to GH Pages + +on: + # Runs on pushes targeting the default branch + push: + branches: ["main"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + name: Build + runs-on: [self-hosted, linux, x64] + steps: + - uses: actions/checkout@v4 + - name: Setup Pages + id: pages + uses: actions/configure-pages@v5 + - uses: eifinger/setup-rye@v4 + - run: rye sync + - name: Run documentation build + run: rye run docs + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/_build/html + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: [self-hosted, linux, x64] + name: Deploy + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..d215458 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,53 @@ +name: Release +on: + push: + tags: + - '[0-9].[0-9]+.[0-9]+' + +permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + contents: read + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + needs: + - build + runs-on: ubuntu-latest + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fe94ef5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# python generated files +.ruff_cache +.pytest_cache +.mypy_cache +__pycache__/ +*.py[oc] +build/ +dist/ +wheels/ +*.egg-info +.venv + +# lock files +requirements.lock +requirements-dev.lock + +# Sphinx build output +**/_build + +# rye is the primary tool, uv is only used for on-the-fly setups +uv.lock diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4932b6f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.2 + hooks: + - id: ruff-format + name: python format + - id: ruff + alias: ruff-check + name: python lint + args: [--fix] + + - repo: https://github.com/ComPWA/taplo-pre-commit + rev: v0.9.3 + hooks: + - id: taplo-format + # lint fetches schemas online at each call, deactivate for now + - id: taplo-lint diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..56bb660 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12.7 diff --git a/README.md b/README.md index 3bd3505..a275cc8 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,37 @@ -* [ ] Move code -* [ ] Create doc project, be based on ubCode/ubTrace -* [ ] ubCode ubproject.toml -* [ ] CI for docs and tests -* [ ] DNS for codelinks.useblocks.com -* [ ] Deployment on pypi (see Sphinx-Needs) -* [ ] Repo rules (no main pushes / branch protection) -* [ ] Cleanup ubTrace (files, ci, rye commands) +# Sphinx CodeLinks + +A Sphinx extension for discovering, linking, and documenting source code across projects. + +## Features + +- **Source Discovery**: Automatically discover source files in your project +- **Virtual Documentation**: Generate documentation from code without modifying source files +- **Code Linking**: Create intelligent links between code elements +- **Sphinx Integration**: Seamless integration with existing Sphinx documentation + +## Quick Start + +```bash +pip install sphinx-codelinks +``` + +Add to your `conf.py`: + +```python +extensions = ['sphinx_needs', 'sphinx_codelinks'] +``` + +## Documentation + +Full documentation: https://codelinks.useblocks.com + +## Components + +- **Source Discovery** ([`src/sphinx_codelinks/source_discovery`](src/sphinx_codelinks/source_discovery)): Code analysis and discovery +- **Virtual Docs** ([`src/sphinx_codelinks/virtual_docs`](src/sphinx_codelinks/virtual_docs)): Documentation generation +- **Sphinx Extension** ([`src/sphinx_codelinks/sphinx_extension`](src/sphinx_codelinks/sphinx_extension)): Sphinx integration +- **Command Line** ([`src/sphinx_codelinks/cmd.py`](src/sphinx_codelinks/cmd.py)): CLI interface + +## Development + +See [Development Guide](docs/source/development/) for contributing guidelines. diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..eca06af --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,67 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +from datetime import datetime +from pathlib import Path +import tomllib + +_project_data = tomllib.loads( + (Path(__file__).parent.parent / "pyproject.toml").read_text("utf8") +)["project"] + +project = _project_data["name"] +author = _project_data["authors"][0]["name"] +copyright = f"{datetime.now().year}, {author}" +version = release = _project_data["version"] + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx_design", + "sphinx_needs", + "sphinx_codelinks", + "sphinx.ext.intersphinx", + "sphinx_code_tabs", + "sphinxcontrib.typer", +] + +# exclude_patterns = [] +templates_path = ["_templates"] +show_warning_types = True + +todo_include_todos = True + +# -- Options for intersphinx extension --------------------------------------- + +intersphinx_mapping = { + "needs": ("https://sphinx-needs.readthedocs.io/en/latest/", None), + "sphinx": ("https://www.sphinx-doc.org/en/master", None), +} + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_title = "CodeLinks" +html_theme = "furo" +# original source is in ubdocs repo at docs/developer_handbook/design/files/ubcode_favicon/favicon.ico +html_favicon = "source/_static/favicon.ico" +html_static_path = ["source/_static"] + +html_theme_options = { + "sidebar_hide_name": True, + "top_of_page_buttons": ["view", "edit"], + "source_repository": "https://github.com/useblocks/sphinx-codelinks", + "source_branch": "main", + "source_directory": "docs/source/", + "light_logo": "sphinx-codelinks-logo_dark.svg", + "dark_logo": "sphinx-codelinks-logo_light.svg", +} +html_css_files = ["furo.css"] + +src_trace_config_from_toml = "./src_trace.toml" diff --git a/docs/source/_static/favicon.ico b/docs/source/_static/favicon.ico new file mode 100644 index 0000000..39dce04 Binary files /dev/null and b/docs/source/_static/favicon.ico differ diff --git a/docs/source/_static/furo.css b/docs/source/_static/furo.css new file mode 100644 index 0000000..8e846ee --- /dev/null +++ b/docs/source/_static/furo.css @@ -0,0 +1,331 @@ +/* Styling for the https://github.com/pradyunsg/furo theme. */ + +:root { + --ub-color-neutral-0: #FFFFFF; + --ub-color-neutral-50: #FAFAFA; + --ub-color-neutral-100: #F5F5F5; + --ub-color-neutral-200: #EEEEEE; + --ub-color-neutral-300: #E0E0E0; + --ub-color-neutral-400: #BDBDBD; + --ub-color-neutral-500: #9E9E9E; + --ub-color-neutral-600: #757575; + --ub-color-neutral-700: #616161; + --ub-color-neutral-800: #424242; + --ub-color-neutral-900: #212121; + --ub-color-neutral-1000: #000000; +} + +/* furo light colors */ +body { + --ub-color-brand-main: #583eff; + --ub-color-brand-muted: #b7a3ff; + --ub-color-brand-opaque: rgb(88, 62, 255, 0.2); + --sn-architecture-bg: url(../architecture_bg-light.png); + + --color-brand-primary: var(--ub-color-brand-main); + + /* anchored heading title */ + --color-highlight-on-target: var(--ub-color-brand-opaque); + + /* Left ToC */ + --color-sidebar-brand-text: var(--color-foreground-primary); + --color-sidebar-caption-text: var(--ub-color-neutral-900); + --color-sidebar-link-text--top-level: var(--ub-color-neutral-800); + --color-sidebar-link-text: var(--ub-color-neutral-600); + --color-sidebar-link-text--current: var(--ub-color-brand-main); + --color-sidebar-item-background--hover: var(--ub-color-brand-opaque); + + /* Right ToC */ + --color-toc-item-text--active: var(--ub-color-brand-main); + + /* Links */ + --color-link: var(--color-content-foreground); + --color-link--hover: var(--color-content-foreground); + --color-link-underline: var(--ub-color-brand-muted); + --color-link-underline--hover: var(--ub-color-brand-main); + --color-link--visited: var(--color-content-foreground); + --color-link--visited--hover: var(--color-content-foreground); + --color-link-underline--visited: var(--ub-color-brand-muted); + --color-link-underline--visited--hover: var(--ub-color-brand-main); + + /* Admonitions */ + --color-admonition-title-background--note: var(--ub-color-brand-opaque); + --color-admonition-title--note: var(--ub-color-brand-main); + + /* Sphinx Design */ + --sd-fontsize-dropdown: var(--admonition-font-size); + --sd-fontsize-dropdown-title: var(--admonition-title-font-size); + --sd-fontweight-dropdown-title: 500; + --sd-color-card-header: var(--ub-color-brand-opaque); + --sd-color-card-border-hover: var(--ub-color-brand-opaque); +} + +/* furo dark colors */ +@media not print { + body[data-theme="dark"] { + + --ub-color-brand-main: #e4ff3e; + --ub-color-brand-muted: #b3bb00; + --ub-color-brand-opaque: rgba(228, 255, 62, 0.15); + --sn-architecture-bg: url(../architecture_bg-dark.png); + + --color-brand-primary: var(--ub-color-brand-main); + + /* anchored heading title */ + --color-highlight-on-target: var(--ub-color-brand-opaque); + + /* Left ToC */ + --color-sidebar-brand-text: var(--color-foreground-primary); + --color-sidebar-caption-text: var(--ub-color-neutral-100); + --color-sidebar-link-text--top-level: var(--ub-color-neutral-300); + --color-sidebar-link-text: var(--ub-color-neutral-500); + --color-sidebar-link-text--current: var(--ub-color-brand-main); + --color-sidebar-item-background--hover: var(--ub-color-brand-opaque); + + /* Right ToC */ + --color-toc-item-text--active: var(--ub-color-brand-main); + + /* Links */ + --color-link: var(--color-content-foreground); + --color-link--hover: var(--color-content-foreground); + --color-link-underline: var(--ub-color-brand-muted); + --color-link-underline--hover: var(--ub-color-brand-main); + --color-link--visited: var(--color-content-foreground); + --color-link--visited--hover: var(--color-content-foreground); + --color-link-underline--visited: var(--ub-color-brand-muted); + --color-link-underline--visited--hover: var(--ub-color-brand-main); + + /* Admonitions */ + --color-admonition-title-background--note: var(--ub-color-brand-opaque); + --color-admonition-title--note: var(--ub-color-brand-main); + } + + @media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) { + + --ub-color-brand-main: #e4ff3e; + --ub-color-brand-muted: #b3bb00; + --ub-color-brand-opaque: rgba(228, 255, 62, 0.15); + --sn-architecture-bg: url(../architecture_bg-dark.png); + + --color-brand-primary: var(--ub-color-brand-main); + + /* anchored heading title */ + --color-highlight-on-target: var(--ub-color-brand-opaque); + + /* Left ToC */ + --color-sidebar-brand-text: var(--color-foreground-primary); + --color-sidebar-caption-text: var(--ub-color-neutral-100); + --color-sidebar-link-text--top-level: var(--ub-color-neutral-300); + --color-sidebar-link-text: var(--ub-color-neutral-500); + --color-sidebar-link-text--current: var(--ub-color-brand-main); + --color-sidebar-item-background--hover: var(--ub-color-brand-opaque); + + /* Right ToC */ + --color-toc-item-text--active: var(--ub-color-brand-main); + + /* Links */ + --color-link: var(--color-content-foreground); + --color-link--hover: var(--color-content-foreground); + --color-link-underline: var(--ub-color-brand-muted); + --color-link-underline--hover: var(--ub-color-brand-main); + --color-link--visited: var(--color-content-foreground); + --color-link--visited--hover: var(--color-content-foreground); + --color-link-underline--visited: var(--ub-color-brand-muted); + --color-link-underline--visited--hover: var(--ub-color-brand-main); + + /* Admonitions */ + --color-admonition-title-background--note: var(--ub-color-brand-opaque); + --color-admonition-title--note: var(--ub-color-brand-main); + } + } +} + +/* sphinx-needs colors */ +/* doc config start */ +/* Note, the recommended way to set colors for furo is in the `html_theme_options` +https://pradyunsg.me/furo/customisation/#light-css-variables-dark-css-variables + +But here we are setting the colors directly in the CSS, +to make it a little easier to compare to the different themes. +*/ +body { + --color-code-background: #eeffcc; + --color-code-foreground: black; + --sn-color-need-border: #555; + --sn-color-need-row-border: hsla(232, 75%, 95%, 0.12); + --sn-color-need-bg: #eee; + --sn-color-need-bg-head: rgba(0, 0, 0, 0.1); + --sn-color-complete-bg-head: rgba(0, 0, 0, 0.1); + --sn-color-complete-bg-foot: rgba(0, 0, 0, 0.1); + --sn-color-bg-gray: #eee; + --sn-color-bg-lightgray: rgba(0, 0, 0, 0.004); + --sn-color-bg-green: #05c46b; + --sn-color-bg-red: #ff3f34; + --sn-color-bg-yellow: #ffc048; + --sn-color-bg-blue: #0fbcf9; + --sn-color-debug-btn-border: #333; + --sn-color-debug-btn-on-text: #f43333; + --sn-color-debug-btn-off-text: #096285; + --sn-color-datatable-label: var(--color-foreground-muted); + --sn-color-datatable-btn-border: var(--color-foreground-muted); +} + +@media not print { + body[data-theme="dark"] { + --color-code-background: #202020; + --color-code-foreground: #d0d0d0; + --sn-color-need-border: #aaaaaa; + --sn-color-need-row-border: hsla(52, 75%, 5%, 0.12); + --sn-color-need-bg: #111111; + --sn-color-need-bg-head: rgba(255, 255, 255, 0.1); + --sn-color-complete-bg-head: rgba(255, 255, 255, 0.1); + --sn-color-complete-bg-foot: rgba(255, 255, 255, 0.1); + --sn-color-bg-gray: #111111; + --sn-color-bg-lightgray: rgba(255, 255, 255, 0.1); + --sn-color-bg-green: #024e2a; + --sn-color-bg-red: #81201b; + --sn-color-bg-yellow: #a97c32; + --sn-color-bg-blue: #096285; + --sn-color-debug-btn-border: #888; + --sn-color-debug-btn-on-text: #ff3f34; + --sn-color-debug-btn-off-text: #0fbcf9; + } + + @media (prefers-color-scheme: dark) { + body:not([data-theme="light"]) { + --color-code-background: #202020; + --color-code-foreground: #d0d0d0; + --sn-color-need-border: #aaaaaa; + --sn-color-need-row-border: hsla(52, 75%, 5%, 0.12); + --sn-color-need-bg: #111111; + --sn-color-need-bg-head: rgba(255, 255, 255, 0.1); + --sn-color-complete-bg-head: rgba(255, 255, 255, 0.1); + --sn-color-complete-bg-foot: rgba(255, 255, 255, 0.1); + --sn-color-bg-gray: #111111; + --sn-color-bg-lightgray: rgba(255, 255, 255, 0.1); + --sn-color-bg-green: #024e2a; + --sn-color-bg-red: #81201b; + --sn-color-bg-yellow: #a97c32; + --sn-color-bg-blue: #096285; + --sn-color-debug-btn-border: #888; + --sn-color-debug-btn-on-text: #ff3f34; + --sn-color-debug-btn-off-text: #0fbcf9; + } + } +} + +/* doc config end */ + +/* make the left ToC use the brand color for the current page */ +.sidebar-tree .current-page>.reference { + font-weight: 700; + color: var(--ub-color-brand-main) +} + +/* styling fo the icon at the top of the left ToC bar */ +img.sidebar-logo { + /* furo sets this at 100% but that makes it a bit too big */ + max-width: 85%; +} + +/* for sphinxcontrib.video */ +video { + width: 700px; + max-width: 100%; +} + +/* Do not underline links in the search results */ +#search-results a { + text-decoration: none; +} + +/* styling for added the source link component in the left ToC bar */ +.gh-source { + display: flex; + align-items: center; + gap: .5em; + padding-left: var(--sidebar-item-spacing-horizontal); + padding-right: var(--sidebar-item-spacing-horizontal); + padding-top: .6em; + padding-bottom: .6em; + text-decoration: none; + border-top: 1px solid var(--color-background-border); + border-bottom: 1px solid var(--color-background-border); +} + +.gh-source--icon { + height: 1.5em; +} + +.gh-source:hover .gh-source--info * { + color: var(--color-foreground-primary); +} + +.gh-source--info { + display: inline-flex; + flex-direction: column; + gap: .1em; +} + +.gh-source--version { + display: inline-flex; + align-items: center; + gap: .2em; +} + +.gh-source--version-icon { + height: .8em; +} + +.gh-source--version-icon, +.gh-source--version-text, +.gh-source--repo-text { + font-size: .8em; + color: var(--color-foreground-muted); +} + + +/** styling for the flowchart diagram on the landing page **/ +svg.sn-flow-chart path.text { + fill: var(--color-foreground-primary); +} + +svg.sn-flow-chart rect.box { + stroke: var(--color-foreground-border); +} + +svg.sn-flow-chart path.arrow { + fill: var(--ub-color-brand-main); +} + +/* Image width fix in need-sidebars. */ +tbody div.needs_side img.needs_image { + max-width: 100px; +} + +/** sphinx-design additional styling **/ +svg.fill-primary { + fill: var(--sd-color-primary); +} + +details.sd-dropdown { + margin: 1rem auto; +} + +summary.sd-summary-title { + padding-right: .5em !important; + /* note this can be removed in sphinx-design v0.6.1 */ +} + +.sn-dropdown-default .sd-summary-icon svg { + color: var(--color-admonition-title--note); +} + +.sn-dropdown-default { + border-left: .2rem solid var(--color-admonition-title--note) !important; +} + +.sn-dropdown-default .sd-summary-title { + border-width: 0 !important; +} diff --git a/sphinx-codelinks-logo_dark.svg b/docs/source/_static/sphinx-codelinks-logo_dark.svg similarity index 100% rename from sphinx-codelinks-logo_dark.svg rename to docs/source/_static/sphinx-codelinks-logo_dark.svg diff --git a/sphinx-codelinks-logo_light.svg b/docs/source/_static/sphinx-codelinks-logo_light.svg similarity index 100% rename from sphinx-codelinks-logo_light.svg rename to docs/source/_static/sphinx-codelinks-logo_light.svg diff --git a/docs/source/basics/installation.rst b/docs/source/basics/installation.rst new file mode 100644 index 0000000..2dba60d --- /dev/null +++ b/docs/source/basics/installation.rst @@ -0,0 +1,23 @@ +.. _installation: + +Installation +============ + +Using Pip +--------- + +.. code-block:: bash + + pip install sphinx-codelinks + +Activation +---------- + +For activation, please add ``sphinx_needs`` and ``sphinx_codelinks`` to the project's extension list in your **conf.py** file + +.. code-block:: python + + extensions = [ + 'sphinx_needs', + 'sphinx_codelinks' + ] diff --git a/docs/source/basics/introduction.rst b/docs/source/basics/introduction.rst new file mode 100644 index 0000000..9066e39 --- /dev/null +++ b/docs/source/basics/introduction.rst @@ -0,0 +1,18 @@ +Introduction +============ + +``CodeLinks`` is a sphinx extension that provides a directive ``src-trace`` +to trace the :external+needs:doc:`Sphinx-Needs ` need items defined in source files. + +Instead of putting RST syntax in the comment, the need definition in source code is simplified to one-liner only, +so that users can just write their `customized one-line comment `_ to have the traceability +from the link between source code and documentation. + +The provided directive leverages the other two modules ``SourceDiscovery`` and ``VirtualDocs``, +which are also packed in the extension, +to discover source files and create the virtual documents for ``src-trace`` to consume. + +Both ``SourceDiscovery`` and ``VirtualDocs`` provide the followings for the developers : + +- **Python API** to extend other further use cases. +- **CLI** to have atomic steps in CI/CD pipelines. diff --git a/docs/source/basics/quickstart.rst b/docs/source/basics/quickstart.rst new file mode 100644 index 0000000..d21ca8e --- /dev/null +++ b/docs/source/basics/quickstart.rst @@ -0,0 +1,74 @@ +Quick Start +=========== + +``CodeLinks`` provides ``src-trace`` directive and it can be used in the following ways: + +.. code-block:: rst + + .. src-trace:: example_with_file + :project: project_config + :file: example.cpp + +or + +.. code-block:: rst + + .. src-trace:: example_with_directory + :project: project_config + :directory: ./example + +``src-trace`` directive has the following options: + +* **project**: the project config specified in ``conf.py`` or ``toml`` file to be used for source tracing. +* **file**: the source file to be traced. +* **directory**: the source files in the directory to be traced recursively. + +Regarding the **file** and **directory** options: + +- they are optional and mutually exclusive. +- the given paths are relative to ``src_dir`` defined in the source tracing configuration +- if not given, the whole project will be examined. + +Example +------- + +With the following configuration for a demo source code project `dcdc `_, + +.. code-block:: python + :caption: conf.py + + src_trace_config_from_toml = "src_trace.toml" + +.. literalinclude:: ./../../src_trace.toml + :caption: src_trace.toml + :language: toml + +``src-trace`` directive can be used with **file** option: + +.. code-block:: rst + + .. src-trace:: dcdc demo_1 + :project: dcdc + :file: ./charge/demo_1.cpp + +The needs defined in source code are extracted and rendered to: + +.. src-trace:: dcdc demo_1 + :project: dcdc + :file: ./charge/demo_1.cpp + +``src-trace`` directive can be used with **directory** option: + +.. code-block:: rst + + .. src-trace:: dcdc charge + :project: dcdc + :directory: ./discharge + +The needs defined in source code are extracted and rendered to: + +.. src-trace:: dcdc charge + :project: dcdc + :directory: ./discharge + +To have a more customized configuration of ``CodeLinks``, please refer to :ref:`configuration `. diff --git a/docs/source/components/cli.rst b/docs/source/components/cli.rst new file mode 100644 index 0000000..e838d9b --- /dev/null +++ b/docs/source/components/cli.rst @@ -0,0 +1,15 @@ +Command Line Interface (CLI) +============================ + +``Sphinx-CodeLinks`` provides CLI for users to integrate documentation build into CI/CD pipeline +and for local development. + +It features help pages. add ``-h`` or ``--help`` to any command to see the available options. + +.. typer:: sphinx_codelinks.cmd.app + :prog: codelinks + :width: 85 + :preferred: svg + :theme: monokai + :show-nested: + :make-sections: diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst new file mode 100644 index 0000000..ded60a3 --- /dev/null +++ b/docs/source/components/configuration.rst @@ -0,0 +1,361 @@ +.. _configuration: + +Configuration +============= + +The configuration for ``CodeLinks`` take place in the project's :external+sphinx:ref:`conf.py file `. + +Each source code project may have different configurations because of its programming language or its locations. +Therefore, based on such consideration, there are **global options** and **project-specific options** for ``CodeLinks`` + +All configuration options starts with the prefix ``src_trace_`` for **Sphinx-CodeLinks**. + +Global Options +-------------- + +The options starts with the prefix ``src_trace_`` are globally applied in the scope of Sphinx documentation. + +src_trace_config_from_toml +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This configuration takes the (relative) path to a `toml file `__ +which contains some or all of the ``CodeLinks`` configuration +(configuration in the toml will override that in the :file:`conf.py`). + +.. code-block:: python + + # Specify the config path for source tracing in conf.py + src_trace_config_from_toml = "src_trace.toml" + +Configuration in the toml can contain any of the following options, under a ``[src_trace]`` section, +but with the ``src_trace_`` prefix removed. + +.. caution:: Any configuration specifying relative paths in the toml file will be resolved relatively to the directory containing the ``toml`` file. + +.. _`src_trace_set_local_url`: + +src_trace_set_local_url +~~~~~~~~~~~~~~~~~~~~~~~ + +Set this option to ``False``, if the local link between a need to the local source code where it is defined is not required. + +Default: **True** + +.. tabs:: + + .. code-tab:: python + + src_trace_set_local_url = True + + .. code-tab:: toml + + [src_trace] + set_local_url = true + +src_trace_set_local_field +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This option is only required if :ref:`src_trace_set_local_url` is set to **True**. + +Set the desired custom field name for the local link to the source code. + +Default: **local-url** + +.. tabs:: + + .. code-tab:: python + + src_trace_local_url_field = "local-url" + + .. code-tab:: toml + + [src_trace] + local_url_field = "local-url" + +.. _`src_trace_set_remote_url`: + +src_trace_set_remote_url +~~~~~~~~~~~~~~~~~~~~~~~~ + +Set this option to ``False``, if the remote link between a need to the remote source code +where it is defined is not required. + +The remote means where the source code is hosted such as GitHub. + +Default: **True** + +.. tabs:: + + .. code-tab:: python + + src_trace_set_remote_url = True + + .. code-tab:: toml + + [src_trace] + set_remote_url = true + +src_trace_set_remote_field +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. note:: This option is only required if :ref:`src_trace_set_remote_url` is set to **True**. + +Set the desired custom field name for the remote link to the source code. + +Default: **remote-url** + +.. tabs:: + + .. code-tab:: python + + src_trace_remote_url_field = "remote-url" + + .. code-tab:: toml + + [src_trace] + remote_url_field = "remote-url" + +Project Specific Options +------------------------ + +Options defined in **src_trace_projects** are project-specific. + +src_trace_projects +~~~~~~~~~~~~~~~~~~ + +This option contains multiple sets of project-specific options. The project name is defined as the key in a dictionary +and its corresponding value is a dictionary containing the options specific to that project. + +.. tabs:: + + .. code-tab:: python + + project_options = dict() + src_trace_projects = { + "project_name": project_options + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + # Project configuration for "project_name" shall be written here + +comment_type +~~~~~~~~~~~~ + +The option defined the comment type used in source code of the project. + +Default: **cpp** + +.. note:: Currently, only C/C++ is supported + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "comment_type": "c" + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + comment_type = "c" + +src_dir +~~~~~~~ + +The relative path from the ``conf.py`` or ``.toml`` file to the source code's root directory + +Default: **./** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "src_dir": "./../src" + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + src_dir = "./../src" + +remote_url_pattern +~~~~~~~~~~~~~~~~~~ + +This option only works with :ref:`src_trace_set_remote_url` set to **True**. +The pattern to access the source code to the remote repositories such as GitHub. + +Default: **Not set** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" + +This option leverages the configuration of :external+needs:ref:`need_string_links` +with the following setup: + +.. code-block:: python + + remote_url_pattern = remote_url_pattern.format( + commit=commit_id, + path=f"{remote_src_dir}/" + "{{value}}", + line="{{lineno}}", + ) + + { + "regex": r"^(?P.+)#L(?P.*)?", + "link_url": remote_url_pattern, + "link_name": "{{value}}#L{{lineno}}", + "options": [remote_url_field], + } + +exclude +~~~~~~~ + +The option is a list of glob patterns to exclude the files which are not required to be addressed + +Default: **[]** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "exclude": ["dcdc/src/ubt/ubt.cpp"] + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + exclude = ["dcdc/src/ubt/ubt.cpp"] + +include +~~~~~~~ + +The option is a list of glob patterns to include the files which are required to be addressed + +Default: **[]** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = + { + "project_name": { + "include": ["dcdc/src/ubt/ubt.cpp"] + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + include = ["dcdc/src/ubt/ubt.cpp"] + +.. note:: **include** option has the highest priority over **exclude** and **gitignore** options. + +gitignore +~~~~~~~~~ + +The option to respect the .gitignore file. + +Default: **True** + +.. tabs:: + + .. code-tab:: python + + src_trace_projects = { + "project_name": { + "gitignore": False + } + + .. code-tab:: toml + + [src_trace.projects.project_name] + gitignore = false + +.. attention:: This option currently does NOT support nested .gitignore files + +.. _`oneline_comment_style`: + +oneline_comment_style +~~~~~~~~~~~~~~~~~~~~~ + +This option enables users to simply define a customized one-line-pattern comment to represent +``Sphinx-Needs`` need items instead of using RST. + +Default: + +.. tabs:: + + .. code-tab:: python + + import os + src_trace_projects = { + "project_name": { + "oneline_comment_style": { + "start_sequence": "@", + "end_sequence": os.linesep, + "field_split_char": ",", + needs_fields = [ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ] + } + } + } + + .. code-tab:: toml + + [src_trace.projects.project_name.oneline_comment_style] + start_sequence = "@" + # end_sequence for the online comments; default is an os-dependant newline character + field_split_char = "," + needs_fields = [ + { "name" = "title", "type" = "str" }, + { "name" = "id", "type" = "str" }, + { "name" = "type", "type" = "str", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [] }, + ] + +With the default, the following one-line comment will be extracted by ``CodeLinks`` and +it is equivalent to the following RST + +.. tabs:: + + .. code-tab:: c + + // @Function Bar, IMPL_4, impl, [SPEC_1, SPEC_2] + + .. code-tab:: RST + + .. impl:: Function Bar + :id: IMPL_4 + :links: [SPEC_1, SPEC_2] + +.. caution:: **type** and **title** must be configured in **needs_fields** as they are mandatory for Sphinx-Needs + +More uses cases can be found in `tests `__ diff --git a/docs/source/components/oneline.rst b/docs/source/components/oneline.rst new file mode 100644 index 0000000..1867d78 --- /dev/null +++ b/docs/source/components/oneline.rst @@ -0,0 +1,218 @@ +.. _oneline: + +One Line Comment Style +====================== + +Many users have raised concerns about the complexity of defining ``Sphinx-Needs`` need items with RST in source code. +Therefore, ``CodeLinks`` provides a customizable one-line comment style pattern to define ``a need items`` +to simplify the effort required to create a need in source code. + +:ref:`Here ` is the default one-line comment style. + +Start and End sequences +----------------------- + +To have better understanding of its the syntax of one-line comment, we will break it down to the following: + +**start_sequence** defines the characters where the one-line comment starts. +**end_sequence** defines the characters where the one-line comment ends. + +The text between **start_sequence** and **end_sequence** contains the fields of ``need items`` + +field_split_char +---------------- + +Since there are always multiple fields for a need, + +**field_split_char** defines the character to split the text into multiple ``pieces/fields``. + +needs_fields +------------ + +Each field in a need may have different data types. +It could be a string if it is a field for ``id`` or ``title``. On the other hand, +it could be a list of strings as well, if the field requires a list of strings to represent ``links``. + +This is where **needs_fields** comes in. + +**needs_fields** contains the fields that are required for needs: + +Each need field defines its: + +- name +- data type (Optional) +- default value (Optional) + +The examples in the following sections use :ref:`the default ` to +explain the syntax of the one-line comment. + +DataType +~~~~~~~~ + +By default, a field has the data type of ``str``. + +For example, if the field definition is as follows: + +.. code-block:: python + + { + "name": "title + } + +It's equivalent to: + +.. code-block:: python + + { + "name": "title", + "type": "str" + } + +If the field is expected to have a list of strings, it shall be defined as the following: + +.. code-block:: python + + { + "name": "links", + "type": "list[str]" + } + +When the field has data type ``list[str]``: + +- the strings must be given within ``[`` and ``]`` brackets +- ``,`` shall be used as the separator. + +For example, with the following **needs_fields** configuration: + +.. _`fields_config`: + +.. code-block:: python + + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + +the online line comment shall be defined as the following + +.. tabs:: + + .. code-tab:: c + + // @ title, id_123, implementation, [link1, link2] + + .. code-tab:: rst + + .. implementation:: title + :id: id_123 + :links: link1, link2 + +Default value +~~~~~~~~~~~~~ + +The value mapped to the key ``default`` in a need field definition is the default value of a need field +when it is not given in the need definition. + +For example, with the following needs_fields definition, + +.. code-block:: python + + needs_fields = [ + { + "name": "title" + }, + { + "name": "type", + "default": "implementation" + }, + ] + +the following need definition in source code is equivalent to RST shown below: + +.. tabs:: + + .. code-tab:: c + + // @ title here and default is used for type + + .. code-tab:: rst + + .. implementation:: title here and default is used for type + +Positional Fields +~~~~~~~~~~~~~~~~~ + +All of the fields defined in ``needs_fields`` are positional fields. +This means the ``order of needs_fields`` determines ``the position of the field`` in the one-line comment. + +For example, with the mentioned :ref:`needs_fields definition ` + +field ``title`` is the first element is the list, so the string of the title must be +the first field in the one-line comment + +.. tabs:: + + .. code-tab:: c + + // @ this is title, this is id, this_type, [link1, link2] + + .. code-tab:: rst + + .. this_type:: this is title + :id: this is id + :links: link1, link2 + +.. note:: A field without a default value cannot follow a field that has a default value set. + +Escaping Characters +~~~~~~~~~~~~~~~~~~~ + +If the value of the field contains characters that are ``field_split_char`` or angular brackets ``[`` and ``]``, + +a leading character ``\`` must be used to escape them. + +For example, with the mentioned :ref:`needs_fields definition `, +``,`` is escaped with ``\`` and is not considered as a separator. + +.. tabs:: + + .. code-tab:: c + + // @ title\, 3, IMPL_3 , impl, [] + + .. code-tab:: rst + + .. impl:: title, 3 + :id: IMPL_3 + +The other example shows the angular brackets ``[`` and ``]`` and comma being escaped: + +.. tabs:: + + .. code-tab:: c + + // @ title 3, IMPL_3 , impl, [\[SPEC\,_1\]] + + .. code-tab:: rst + + .. impl:: title 3 + :id: IMPL_3 + :links: [SPEC,_1] + +To have a backslash ``\`` as a literal in the value, use ``\\`` as shown in the following: + +.. tabs:: + + .. code-tab:: c + + // @ title\\ 3, IMPL_3 , impl, [\[SPEC\,_1\]] + + .. code-tab:: rst + + .. impl:: title\ 3 + :id: IMPL_3 + :links: [SPEC,_1] + +.. caution:: Field values can never contain any newline characters ``\r`` or ``\n``. diff --git a/docs/source/development/change_log.rst b/docs/source/development/change_log.rst new file mode 100644 index 0000000..c1ff73d --- /dev/null +++ b/docs/source/development/change_log.rst @@ -0,0 +1,17 @@ +.. _changelog: + +Changelog +========= + +0.1.0 +----- + +:Released: 11.07.2025 + +Initial release of ``Sphinx-CodeLinks`` + +This version features: + +- Sphinx Directive ``src-trace`` +- Virtual Docs and Source Discovery CLI +- One-line comment to define a ``Sphinx-Needs`` need item diff --git a/docs/source/development/contributing.rst b/docs/source/development/contributing.rst new file mode 100644 index 0000000..03b3128 --- /dev/null +++ b/docs/source/development/contributing.rst @@ -0,0 +1,72 @@ +Contributing +============ + +This page provides a guide for developers wishing to contribute to ``Sphinx-CodeLinks``. + +Bugs, Features and PRs +---------------------- + +For **bug reports** and well-described **technical feature request**, please use our issue tracker: +https://github.com/useblocks/sphinx-codelinks/issues + +If you have already created a PR, you can send it in. Our CI workflow will check (test and code styles) +and a maintainer will perform a review, before we can merge it. +Your PR should conform with the following rules: + +- A meaningful description or link, which describes the change +- The changed code (for sure :) ) +- Test cases for the change (important!) +- Updated documentation, if behavior gets changed or new options/directives are introduced. +- Update of docs/changelog.rst. + +Install Dependencies +-------------------- + +``CodeLinks`` uses `rye `_ to manage the repository. + +For the development, use the following command to install python dependencies into the virtual environment. + +.. code-block:: bash + + rye sync + +Formatting, Linting and Typing +------------------------------ + +To run the formatting and linting, pre-commit is used: + +.. code-block:: bash + + pre-commit install # to auto-run on every commit + pre-commit run --all-files # to run manually + +The CI also checks typing, use the following command locally to see if your code is well-typed + +.. code-block:: bash + + rye run mypy:all + +Build docs +---------- + +To build the documentation stored in ``docs``, run: + +.. code-block:: bash + + rye run docs + +Test Cases +---------- + +To run test cases locally: + +.. code-block:: bash + + rye test -a + +Note some tests use `syrupy `__ to perform snapshot testing. +These snapshots can be updated by running: + +.. code-block:: bash + + pytest tests/ --snapshot-update diff --git a/docs/source/development/roadmap.rst b/docs/source/development/roadmap.rst new file mode 100644 index 0000000..b21529a --- /dev/null +++ b/docs/source/development/roadmap.rst @@ -0,0 +1,33 @@ +.. _roadmap: + +Roadmap +======= + +Other Comment styles +-------------------- + +Currently, only ``C/C++`` comment style is supported. +The other comment styles for different programming languages are planed, such as: + +- Python +- Rust +- YAML +- SyML + +Nested .gitignore +----------------- + +``CodeLinks`` respects ``.gitignore`` file, but if the .gitignore files are nested, it's not supported. +Respecting nested ``.gitignore`` in the context of the git repositories is planned. + +Flexible way to define Sphinx-Needs need items in source code +------------------------------------------------------------- + +The only way to define ``Sphinx-Needs`` need items is through ``one-line comment style``. +Raw RST text and multi-lines comments style are planned to support + +Export needs.json +----------------- + +To facilitate CI workflow and enhance the portability of ``need items`` defined in source code, +we plan to have the feature to export the needs defined in source code to a JSON file. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..710e85b --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,70 @@ +.. grid:: + :class-row: sd-w-100 + + .. grid-item:: + :columns: 12 8 8 8 + :child-align: justify + :class: sd-fs-3 + + .. div:: sd-font-weight-bold + + The portal to your source code + + .. div:: sd-fs-5 sd-font-italic + + ``Sphinx-CodeLinks`` is designed for Engineering-as-Code workflows to facilitate Application Lifecycle Management (ALM). + It enables users to define ``Sphinx-Needs`` need items within source code using a single line and automatically extract them + into the documentation during the Sphinx build process. + + .. grid:: 1 1 2 2 + :gutter: 2 2 3 3 + :margin: 2 + :padding: 0 + + .. grid-item:: + :columns: auto + + .. button-ref:: basics/installation + :ref-type: doc + :outline: + :color: primary + :class: sd-rounded-pill sd-px-4 sd-fs-5 + + Get Started + + .. grid-item:: + :columns: auto + + .. button-link:: https://useblocks.com/ + :outline: + :color: primary + :class: sd-rounded-pill sd-px-4 sd-fs-5 + + About useblocks + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + :caption: Basics + + basics/introduction + basics/installation + basics/quickstart + +.. toctree:: + :maxdepth: 1 + :caption: Components + + components/configuration + components/oneline + components/cli + +.. toctree:: + :maxdepth: 1 + :caption: Development + + development/roadmap + development/change_log + development/contributing diff --git a/docs/src_trace.toml b/docs/src_trace.toml new file mode 100644 index 0000000..1f68587 --- /dev/null +++ b/docs/src_trace.toml @@ -0,0 +1,29 @@ +[src_trace] +# Configuration for source tracing +set_local_url = true # Set to true to enable local code html and URL generation +local_url_field = "local-url" # Need's field name for local URL +set_remote_url = true # Set to true to enable remote url to be generated +remote_url_field = "remote-url" # Need's field name for remote URL + +[src_trace.projects.dcdc] +# Configuration for source tracing project "dcdc" +comment_type = "cpp" # Type of the comment, only support C/C++ for now +src_dir = "../tests/data/dcdc" # Relative path from conf.py to the source directory +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" # URL pattern for remote source code +exclude = ["dcdc/src/ubt/ubt.cpp"] # Exclude files from source tracing +include = ["**/*.cpp", "**/*.hpp"] # Include files for source tracing +gitignore = true # Respect .gitignore to filter files + +[src_trace.projects.dcdc.oneline_comment_style] +# Configuration for oneline comment style +start_sequence = "[[" # Start sequence for oneline comments +end_sequence = "]]" # End sequence for the online comments; default is newline character +field_split_char = "," # Character to split fields in the comment +# Fields that are defined in the oneline comment style +needs_fields = [ + { "name" = "id", "type" = "str" }, + { "name" = "title", "type" = "str" }, + { "name" = "type", "type" = "str", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, +] diff --git a/docs/ubproject.toml b/docs/ubproject.toml new file mode 100644 index 0000000..5429bcf --- /dev/null +++ b/docs/ubproject.toml @@ -0,0 +1,12 @@ +"$schema" = "https://ubcode.useblocks.com/ubproject.schema.json" + +[rst_lint] +ignore = ["block.title_line"] + +[needs] +id_required = true + +[[needs.types]] +directive = "my-req" +title = "My Requirement" +prefix = "M_" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..09d868e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,172 @@ +[project] +name = "sphinx-codelinks" +version = "0.1.0" +description = "Add your description here" +authors = [{ name = "team useblocks", email = "info@useblocks.com" }] +maintainers = [ + { name = "Marco Heinemann", email = "marco.heinemann@useblocks.com" }, +] +license = { file = "LICENSE" } +readme = "README.md" +requires-python = ">= 3.12" +dependencies = [ + "comment-parser>=1.2.4", + "gitignore-parser>=0.1.11", + "typer>=0.16.0", + "jsonschema", + "sphinx>=7.4,<9", + "sphinx-needs>=4.2.0", + # unconstrained versions, to be pinned by user or Sphinx + "jinja2", + "pygments", + "docutils", # constrained by user or Sphinx +] + +[build-system] +requires = ["flit_core >=3.4,<4"] +build-backend = "flit_core.buildapi" + +[tool.rye] +managed = true +dev-dependencies = [ + "types-docutils", + "types-Pygments", + "syrupy>=4.9.1", + "furo>=2024.5.6", + "moto ~= 5.0", + "mypy>=1.12.1", + "myst-parser>=4.0.0", + "pydantic ~= 2.9", + "pip-licenses>=5.0.0", + "psutil>=7.0.0", + "pytest-cov>=5.0.0", + "pytest>=8.2.2", + "simple-build>=0.0.2", + "sphinx-design>=0.6.1", + "types-psutil>=7.0.0.20250218", + "uv>=0.5.5", + "pytest-docker>=3.1.2", + "shiv>=1.0.8", + "insta-science>=0.2.1", + "types-jsonschema>=4.23.0.20241208", + "toml>=0.10.2", + "sphinx-code-tabs>=0.5.5", + "sphinxcontrib-typer>=0.5.1", +] + +[project.scripts] +codelinks = "sphinx_codelinks.cmd:app" + +[tool.rye.scripts] +# linting and formatting +"mypy:all" = "mypy ." +"rye:lint" = "rye lint" +"rye:format" = "rye format" +"check" = { chain = ["rye:format", "rye:lint", "mypy:all"] } +# docs html +"docs:rm" = "rm -rf docs/_build/html" +"docs" = "sphinx-build -nW --keep-going -b html -T -c docs docs/source docs/_build/html" +"docs:clean" = { chain = ["docs:rm", "docs"] } + +[tool.ruff.lint] +extend-select = [ + # "ANN", + "S", + "ARG", + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "FURB", # refurb (modernising code) + "I", # isort + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "N", # pep8-naming + "PERF", # perflint (performance anti-patterns) + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PTH", # flake8-use-pathlib + "RUF", # Ruff-specific rules + "SIM", # flake8-simplify + "SLF", # private member access + "UP", # pyupgrade + "T20", # flake8-print +] +extend-ignore = [ + "ISC001", # implicit-str-concat +] + +[tool.ruff.lint.isort] +split-on-trailing-comma = false +force-sort-within-sections = true + +[tool.ruff.lint.per-file-ignores] +"**/tests/*" = [ + "ARG001", # unused-function-argument - fixtures + "ARG005", # unused-lambda-argument - monkeypatches + "PLR2004", # magic-value-comparison - valueable for tests + "S101", # assert - needed for tests +] +"**/build_hooks_*/**" = [ + "S607", # start-process-with-partial-path - pyarmor call in rye context + "S603", # subprocess-without-shell-equals-true - pyarmor call +] +"scripts/*.py" = [ + "T201", # print - used for output + "S607", # start-process-with-partial-path - pyarmor call in rye context + "S603", # subprocess-without-shell-equals-true - build scripts +] +"src/sphinx_codelinks/sphinx_extension/debug.py" = [ + "T201", # print - used for output + "UP047", # on-pep695-generic-function - it's generic +] +"src/sphinx_codelinks/cmd.py" = [ + "PLC0415", # import on top - only import relevant modules by use cases +] + +[tool.mypy] +exclude = ["tests/", "dist/", "docs/_build/", "docs/conf.py"] +show_error_codes = true +warn_unused_ignores = true +warn_redundant_casts = true +strict = true +# disallow dynamic typing +disallow_any_unimported = true +disallow_any_expr = true +# disallow_any_decorated = true +disallow_any_explicit = true +disallow_any_generics = true +disallow_subclassing_any = true +# dissallow untyped definitions and calls +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_decorators = true + +plugins = ["pydantic.mypy"] +mypy_path = "typings" + +[[tool.mypy.overrides]] +module = ["licensing.*", "tomlkit.*"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = "tests.*" +disallow_any_explicit = false +disallow_any_unimported = false +disallow_untyped_defs = false +disallow_any_expr = false + +[[tool.mypy.overrides]] +module = "sphinx_codelinks.*" +disallow_any_unimported = false +disallow_untyped_defs = false +disallow_any_expr = false + + +[tool.pydantic-mypy] +init_forbid_extra = true +init_typed = true +warn_required_dynamic_aliases = true + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/src/sphinx_codelinks/README.md b/src/sphinx_codelinks/README.md new file mode 100644 index 0000000..de5c307 --- /dev/null +++ b/src/sphinx_codelinks/README.md @@ -0,0 +1,46 @@ +# CodeLinks + +## Overview + +This is a Sphinx extension to extract Sphinx-Needs items from source files +such as C, C++ and others. + +The need items are defined in the source files as comments and can be used to define +test case specifications or implementation markers. + +Various definition styles are supported, such as one-line, multi-line or raw RST. + +The project consists of the following three components: + +- Source Discovery: determines list of source files from a given directory +- Virtual Docs: extract need annotations while keeping the source map +- Source Tracing: Sphinx extension to represent the collected the needs in the documentation + +`Source Discovery` and `Virtual Docs` can be used as `APIs` or `CLI tools`. +The detail usages can be found in the [test cases](./tests). + +The library is built to be + +- ⚡ fast for large code bases and +- 📃 support a multitude of languages. + +## Source Discovery + +Recursively collect the file paths from a given directory. +It can be configured to respect `.gitignore`. + +## Virtual Docs + +Virtual Docs parses the discoverd files and + +- extracts the need items from the comments in the source files. +- extracts additional metadata such as extra options and links. +- generates virtual documents containing the above-mentioned information into `json` files. +- caches virtual docs for incremental builds. +- keeps the source map to the path and line number of the original source files. + +## CodeLinks + +CodeLinks is a Sphinx Extension based on Sphinx-Needs. It provides the directive `src-tracing` +to collect the needs defined in source files by using `Source Discovery` and `Virtual Docs` +under the hood. diff --git a/src/sphinx_codelinks/__init__.py b/src/sphinx_codelinks/__init__.py new file mode 100644 index 0000000..00b724d --- /dev/null +++ b/src/sphinx_codelinks/__init__.py @@ -0,0 +1,10 @@ +"""CodeLinks source code analyzer""" + +from sphinx_codelinks.sphinx_extension.source_tracing import setup + +__version__ = "0.1.0" + +__all__ = [ + "__version__", + "setup", +] diff --git a/src/sphinx_codelinks/cmd.py b/src/sphinx_codelinks/cmd.py new file mode 100644 index 0000000..41290d8 --- /dev/null +++ b/src/sphinx_codelinks/cmd.py @@ -0,0 +1,212 @@ +from collections import deque +from os import linesep +from pathlib import Path +import tempfile +import tomllib +from typing import Annotated, cast + +import typer + +from sphinx_codelinks.source_discovery.config import SourceDiscoveryConfig +from sphinx_codelinks.sphinx_extension.config import ( + SrcTraceProjectConfigFileType, + SrcTraceProjectConfigType, + build_src_discovery_dict, + validate_oneline_comment_style, +) +from sphinx_codelinks.virtual_docs.config import ( + OneLineCommentStyle, + OneLineCommentStyleType, +) + +app = typer.Typer( + no_args_is_help=True, context_settings={"help_option_names": ["-h", "--help"]} +) + + +@app.command(no_args_is_help=True) +def discover( + root_dir: Annotated[ + Path, + typer.Argument( + ..., + help="Root directory for discovery", + show_default=False, + dir_okay=True, + file_okay=False, + exists=True, + resolve_path=True, + ), + ], + exclude: Annotated[ + list[str] | None, + typer.Option( + "--excludes", + "-e", + help="Glob patterns to be excluded.", + ), + ] = None, + include: Annotated[ + list[str] | None, + typer.Option( + "--includes", + "-i", + help="Glob patterns to be included.", + ), + ] = None, + gitignore: Annotated[bool, typer.Option(help="Respect .gitignore(s)")] = True, + file_types: Annotated[ + list[str] | None, + typer.Option( + "--file-type", + "-f", + help="The file extension to be discovered. If not specified, all files are discovered.", + ), + ] = None, +) -> None: + """Discover the filepaths from the given root directory.""" + from sphinx_codelinks.source_discovery.source_discover import SourceDiscover + + source_discover = SourceDiscover( + root_dir=root_dir, + exclude=exclude, + include=include, + file_types=file_types, + gitignore=gitignore, + ) + typer.echo(f"{len(source_discover.source_paths)} files discovered") + for file_path in source_discover.source_paths: + typer.echo(file_path) + + +@app.command(no_args_is_help=True) +def vdoc( + config: Annotated[ + Path, + typer.Option( + "--config", + "-c", + help="The toml config file", + show_default=False, + dir_okay=False, + file_okay=True, + exists=True, + ), + ], + project: Annotated[ + str | None, typer.Option("--project", "-p", help="project identifier in config") + ] = None, + output_dir: Path = typer.Option( # noqa: B008 # to support filepath + Path(tempfile.gettempdir()), # noqa: B008 # to support filepath + "--output-dir", + "-o", + help="The output directory of generated documents and caches.", + ), +) -> None: + """Generate virtual documents for caching and extract the oneline comments.""" + + data = load_config_from_toml(config, project) + + errors: deque[str] = deque() + + oneline_comment_style_dict: OneLineCommentStyleType | None = data.get( + "oneline_comment_style" + ) + if oneline_comment_style_dict is None: + oneline_comment_style = OneLineCommentStyle() + else: + try: + oneline_comment_style = OneLineCommentStyle(**oneline_comment_style_dict) + except TypeError as e: + raise typer.BadParameter( + f"Invalid oneline comment style configuration: {e}" + ) from e + + project_config = cast( + SrcTraceProjectConfigType, + { + key: value if key != "oneline_comment_style" else oneline_comment_style + for key, value in data.items() + }, + ) + oneline_errors = validate_oneline_comment_style(project_config) + + if oneline_errors: + errors.appendleft("Invalid oneline comment style configuration:") + errors.extend(oneline_errors) + + src_discovery_dict, src_discovery_errors = build_src_discovery_dict(project_config) + if src_discovery_dict: + src_discovery_config = SourceDiscoveryConfig(**src_discovery_dict) + else: + src_discovery_config = SourceDiscoveryConfig() + src_discovery_errors.extend(src_discovery_config.check_schema()) + + if src_discovery_errors: + errors.appendleft("Invalid source discovery configuration:") + errors.extend(src_discovery_errors) + + if errors: + raise typer.BadParameter(f"{linesep.join(errors)}") + + from sphinx_codelinks.source_discovery.source_discover import SourceDiscover + + src_root_dir = (config.parent / src_discovery_config.root_dir).resolve() + source_discover = SourceDiscover( + root_dir=src_root_dir, + exclude=src_discovery_config.exclude, + include=src_discovery_config.include, + file_types=src_discovery_config.file_types, + gitignore=src_discovery_config.gitignore, + ) + + from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs + + virtual_docs = VirtualDocs( + src_files=source_discover.source_paths, + src_dir=str(src_root_dir), + output_dir=str(output_dir), + oneline_comment_style=oneline_comment_style, + comment_type=src_discovery_config.file_types[0] + if src_discovery_config.file_types + else "c", + ) + virtual_docs.collect() + virtual_docs.dump_virtual_docs() + + if len(virtual_docs.virtual_docs) > 0: + typer.echo("The virtual documents are generated:") + for v_doc in virtual_docs.virtual_docs: + json_path = output_dir / v_doc.filepath.with_suffix(".json").relative_to( + src_root_dir + ) + typer.echo(json_path) + else: + typer.echo("No virtual documents are generated.") + + virtual_docs.cache.update_cache() + typer.echo("The cached files are:") + for cached_file in virtual_docs.cache.cached_files: + typer.echo(cached_file) + + +def load_config_from_toml( + toml_file: Path, project: str | None = None +) -> SrcTraceProjectConfigFileType: + try: + with toml_file.open("rb") as f: + toml_data = tomllib.load(f) + + if project: + toml_data = toml_data["src_trace"]["projects"][project] + + except Exception as e: + raise Exception( + f"Failed to load source tracing configuration from {toml_file}" + ) from e + + return cast(SrcTraceProjectConfigFileType, toml_data) + + +if __name__ == "__main__": + app() diff --git a/src/sphinx_codelinks/source_discovery/config.py b/src/sphinx_codelinks/source_discovery/config.py new file mode 100644 index 0000000..e6eb62b --- /dev/null +++ b/src/sphinx_codelinks/source_discovery/config.py @@ -0,0 +1,68 @@ +from dataclasses import MISSING, dataclass, field, fields +from pathlib import Path +from typing import Any, TypedDict, cast + +from jsonschema import ValidationError, validate + + +class SourceDiscoveryConfigType(TypedDict, total=False): + root_dir: Path + exclude: list[str] + include: list[str] + gitignore: bool + file_types: list[str] + + +@dataclass +class SourceDiscoveryConfig: + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + root_dir: Path = field( + default_factory=lambda: Path.cwd(), metadata={"schema": {"type": "string"}} + ) + """The root of the source directory.""" + + exclude: list[str] = field( + default_factory=list, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """The glob pattern to exclude files.""" + + include: list[str] = field( + default_factory=list, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """The glob pattern to include files.""" + + gitignore: bool = field(default=True, metadata={"schema": {"type": "boolean"}}) + """Whether to respect .gitignore to exclude files.""" + + file_types: list[str] = field( + default_factory=lambda: ["c", "h", "cpp", "hpp"], + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """The file types to discover.""" + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[explicit-any] + return None + + def check_schema(self) -> list[str]: + errors = [] + for _field_name in self.field_names(): + schema = self.get_schema(_field_name) + value = getattr(self, _field_name) + if isinstance(value, Path): # adapt to json schema restriction + value = str(value) + try: + validate(instance=value, schema=schema) # type: ignore[arg-type] # validate has no type specified + except ValidationError as e: + errors.append( + f"Schema validation error in field '{_field_name}': {e.message}" + ) + return errors diff --git a/src/sphinx_codelinks/source_discovery/source_discover.py b/src/sphinx_codelinks/source_discovery/source_discover.py new file mode 100644 index 0000000..f4853c4 --- /dev/null +++ b/src/sphinx_codelinks/source_discovery/source_discover.py @@ -0,0 +1,69 @@ +from collections.abc import Callable +import fnmatch +import os +from pathlib import Path + +from gitignore_parser import parse_gitignore # type: ignore[import-untyped] + + +class SourceDiscover: + def __init__( + self, + root_dir: Path, + exclude: list[str] | None = None, + include: list[str] | None = None, + gitignore: bool = True, + file_types: list[str] | None = None, + ): + self.root_path = root_dir + self.exclude = exclude + self.include = include + # Only gitignore at source root is considered. + # TODO: Support nested gitignore files + gitignore_path = self.root_path / ".gitignore" + self.gitignore_matcher: Callable[[str], bool] | None = ( + parse_gitignore(gitignore_path) + if gitignore and gitignore_path.exists() + else None + ) + # normalize the file types to lower case with leading dot + self.file_types = ( + { + file_type.lower() + if file_type.startswith(".") + else f".{file_type}".lower() + for file_type in file_types + } + if file_types + else None + ) + + self.source_paths = self._discover() + + def _discover(self) -> list[Path]: + """Discover source files recursively in the given directory.""" + discovered_files = [] + for filepath in self.root_path.rglob("*"): + if filepath.is_file(): + if self.file_types and filepath.suffix.lower() not in self.file_types: + continue + rel_filepath = str(filepath.relative_to(self.root_path)) + if self.include and self._matches_any(rel_filepath, self.include): + # "includes" has the highest priority over "gitignore" and "excludes" + discovered_files.append(filepath) + continue + if self.gitignore_matcher and self.gitignore_matcher( + str(filepath.absolute()) + ): + continue + if self.exclude and self._matches_any(rel_filepath, self.exclude): + continue + discovered_files.append(filepath) + sorted_filepaths = sorted( + discovered_files, key=lambda x: os.path.normcase(os.path.normpath(x)) + ) + return sorted_filepaths + + def _matches_any(self, rel_filepath: str, patterns: list[str]) -> bool: + """Check if the given file path matches any of the given patterns.""" + return any(fnmatch.fnmatch(rel_filepath, pattern) for pattern in patterns) diff --git a/src/sphinx_codelinks/sphinx_extension/config.py b/src/sphinx_codelinks/sphinx_extension/config.py new file mode 100644 index 0000000..8cdb075 --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/config.py @@ -0,0 +1,307 @@ +from dataclasses import MISSING, dataclass, field, fields +from pathlib import Path +from typing import Any, Literal, TypedDict, cast + +from jsonschema import ValidationError, validate +from sphinx.application import Sphinx +from sphinx.config import Config as _SphinxConfig + +from sphinx_codelinks.source_discovery.config import ( + SourceDiscoveryConfig, + SourceDiscoveryConfigType, +) +from sphinx_codelinks.virtual_docs.config import ( + SUPPORTED_COMMENT_TYPES, + OneLineCommentStyle, + OneLineCommentStyleType, +) + +SRC_TRACE_CACHE: str = "src_trace_cache" + + +class SourceTracingLineHref: + """Global class for the mapping between source file line numbers and Sphinx documentation links.""" + + def __init__(self) -> None: + self.mappings: dict[str, dict[int, str]] = {} + + +file_lineno_href = SourceTracingLineHref() + + +class SrcTraceProjectConfigFileType(TypedDict): + # only support C/C++ for now + comment_type: Literal["cpp", "hpp", "c", "h"] + src_dir: str + remote_url_pattern: str + exclude: list[str] + include: list[str] + gitignore: bool + oneline_comment_style: OneLineCommentStyleType + + +class SrcTraceProjectConfigType(TypedDict): + # only support C/C++ for now + comment_type: Literal["cpp", "hpp", "c", "h"] + src_dir: str + remote_url_pattern: str + exclude: list[str] + include: list[str] + gitignore: bool + oneline_comment_style: OneLineCommentStyle + + +class SrcTraceConfigType(TypedDict): + config_from_toml: str | None + set_local_url: bool + local_url_field: str + set_remote_url: bool + remote_url_field: str + projects: dict[str, SrcTraceProjectConfigType] + debug_measurement: bool + debug_filters: bool + + +@dataclass +class SrcTraceSphinxConfig: + def __init__(self, config: _SphinxConfig) -> None: + super().__setattr__("_config", config) + + def __getattribute__(self, name: str) -> Any: # type: ignore[explicit-any] + if name.startswith("__") or name == "_config": + return super().__getattribute__(name) + return getattr(super().__getattribute__("_config"), f"src_trace_{name}") + + def __setattr__(self, name: str, value: Any) -> None: # type: ignore[explicit-any] + if name == "_config" and "src_trace_projects" in value: + src_trace_projects: dict[str, SrcTraceProjectConfigType] = value[ + "src_trace_projects" + ] + for _config in src_trace_projects.values(): + # overwrite the config into different types on purpose + # covert dict to OneLineCommentStyle class + oneline_comment_style: OneLineCommentStyleType | None = cast( + OneLineCommentStyleType, _config.get("oneline_comment_style") + ) + if not oneline_comment_style: + raise Exception("OneLineCommentStyle is not given") + + _config["oneline_comment_style"] = OneLineCommentStyle( + **oneline_comment_style + ) + if name.startswith("__") or name == "_config": + return super().__setattr__(name, value) + + return setattr(super().__getattribute__("_config"), f"src_trace_{name}", value) + + @classmethod + def add_config_values(cls, app: Sphinx) -> None: + """Add all config values to Sphinx application""" + for item in fields(cls): + if item.default_factory is not MISSING: + default = item.default_factory() + elif item.default is not MISSING: + default = item.default + else: + raise Exception(f"Field {item.name} has no default value or factory") + + name = item.name + app.add_config_value( + f"src_trace_{name}", + default, + item.metadata["rebuild"], + types=item.metadata["types"], + ) + + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] + """Get the schema for a config item.""" + _field = next(field for field in fields(cls) if field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return _field.metadata["schema"] # type: ignore[no-any-return] + return None + + config_from_toml: str | None = field( + default=None, + metadata={ + "rebuild": "env", + "types": (str, type(None)), + "schema": { + "type": ["string", "null"], + "examples": ["config.toml", None], + }, + }, + ) + """Path to a TOML file to load configuration from.""" + + set_local_url: bool = field( + default=False, + metadata={ + "rebuild": "env", + "types": (bool,), + "schema": { + "type": "boolean", + }, + }, + ) + """Set the file URL in the extracted need.""" + + local_url_field: str = field( + default="local-url", + metadata={ + "rebuild": "env", + "types": (str,), + "schema": { + "type": "string", + }, + }, + ) + """The field name for the file URL in the extracted need.""" + + set_remote_url: bool = field( + default=False, + metadata={ + "rebuild": "env", + "types": (bool,), + "schema": { + "type": "boolean", + }, + }, + ) + remote_url_field: str = field( + default="remote-url", + metadata={ + "rebuild": "env", + "types": (str,), + "schema": { + "type": "string", + }, + }, + ) + """The field name for the remote URL in the extracted need.""" + + projects: dict[str, SrcTraceProjectConfigType] = field( + default_factory=dict, + metadata={ + "rebuild": "env", + "types": (), + "schema": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "comment_type": {}, + "src_dir": {}, + "remote_url_pattern": {}, + "exclude": {}, + "include": {}, + "gitignore": {}, + "oneline_comment_style": {}, + }, + "additionalProperties": False, + }, + }, + }, + ) + """The configuration for the source tracing projects.""" + + debug_measurement: bool = field( + default=False, metadata={"rebuild": "html", "types": (bool,)} + ) + """If True, log runtime information for various functions.""" + debug_filters: bool = field( + default=False, metadata={"rebuild": "html", "types": (bool,)} + ) + """If True, log filter processing runtime information.""" + + +def check_schema(config: SrcTraceSphinxConfig) -> list[str]: + errors = [] + for _field_name in SrcTraceSphinxConfig.field_names(): + schema = SrcTraceSphinxConfig.get_schema(_field_name) + value = getattr(config, _field_name) + if not schema: + continue + try: + validate(instance=value, schema=schema) + except ValidationError as e: + errors.append( + f"Schema validation error in filed '{_field_name}': {e.message}" + ) + return errors + + +def check_project_configuration(config: SrcTraceSphinxConfig) -> list[str]: + errors = [] + + for project_name, project_config in config.projects.items(): + project_errors: list[str] = [] + oneline_errors = validate_oneline_comment_style(project_config) + src_discovery_dict, src_discovery_errors = build_src_discovery_dict( + project_config + ) + if src_discovery_dict is not None: + src_discovery_config = SourceDiscoveryConfig(**src_discovery_dict) + src_discovery_errors.extend(src_discovery_config.check_schema()) + + if config.set_remote_url and "remote_url_pattern" not in project_config: + project_errors.append( + "remote_url_pattern must be given, as set_remote_url is enabled" + ) + + if "remote_url_pattern" in project_config and not isinstance( + project_config["remote_url_pattern"], str + ): + project_errors.append("remote_url_pattern must be a string") + + if oneline_errors or src_discovery_errors or project_errors: + errors.append(f"Project '{project_name}' has the following errors:") + errors.extend(oneline_errors) + errors.extend(src_discovery_errors) + errors.extend(project_errors) + + return errors + + +def check_configuration(config: SrcTraceSphinxConfig) -> list[str]: + errors = [] + errors.extend(check_schema(config)) + errors.extend(check_project_configuration(config)) + return errors + + +def validate_oneline_comment_style( + project_config: SrcTraceProjectConfigType, +) -> list[str]: + if "oneline_comment_style" in project_config: + style = project_config["oneline_comment_style"] + if isinstance(style, OneLineCommentStyle): + return style.check_fields_configuration() + return [] + + +def build_src_discovery_dict( + project_config: SrcTraceProjectConfigType, +) -> tuple[SourceDiscoveryConfigType | None, list[str]]: + src_discovery_dict = cast(SourceDiscoveryConfigType, {}) + src_discovery_errors = [] + if "src_dir" in project_config: + if isinstance(project_config["src_dir"], str): + src_discovery_dict["root_dir"] = Path(project_config["src_dir"]) + else: + src_discovery_errors.append("src_dir must be a string") + for key in ("exclude", "include", "gitignore"): + if key in project_config: + src_discovery_dict[key] = project_config[key] + if "comment_type" in project_config: + if project_config["comment_type"] not in SUPPORTED_COMMENT_TYPES: + src_discovery_errors.append( + f"comment_type must be one of {sorted(SUPPORTED_COMMENT_TYPES)}" + ) + else: + src_discovery_dict["file_types"] = list(SUPPORTED_COMMENT_TYPES) + return src_discovery_dict, src_discovery_errors diff --git a/src/sphinx_codelinks/sphinx_extension/debug.py b/src/sphinx_codelinks/sphinx_extension/debug.py new file mode 100644 index 0000000..cf4ef4c --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/debug.py @@ -0,0 +1,189 @@ +""" +Contains debug features to track down +runtime and other problems with Src-Trace +""" + +from __future__ import annotations + +from collections.abc import Callable +from datetime import datetime +from functools import wraps +import inspect +import json +from pathlib import Path +from timeit import default_timer as timer # Used for timing measurements +from typing import Any, TypeVar + +from jinja2 import Environment, PackageLoader, select_autoescape +from sphinx.application import Sphinx + +# Stores the timing results +TIME_MEASUREMENTS: dict[str, Any] = {} # type: ignore[explicit-any] +EXECUTE_TIME_MEASUREMENTS = ( + False # Will be used to de/activate measurements. Set during a Sphinx Event +) + +START_TIME = 0.0 + +T = TypeVar("T", bound=Callable[..., Any]) # type: ignore[explicit-any] + + +def measure_time( # type: ignore[explicit-any] + category: str | None = None, source: str = "internal", name: str | None = None +) -> Callable[[T], T]: + """ + Decorator for measuring the needed execution time of a specific function. + + It measures: + + * Amount of executions + * Overall time consumed + * Average time of an execution as `avg` + * Minimum time of an execution as `min` + * Maximum time of an execution as `max` + + For `max` also the used function parameters are stored as string values, to make + it easier to reproduce the maximum case. + + Usage as decorator:: + + from sphinx_needs.utils import measure_time + + @measure_time('my_category') + def my_cool_function(a, b,c ): + # does something + + :param category: Name of a category, which helps to cluster the measured functions. + :param source: Should be "internal" or "user". Used to easily structure function written by user. + :param name: Name to use for the measured. If not given, the function name is used. + """ + + def inner(func: T) -> T: # type: ignore[explicit-any] + @wraps(func) + def wrapper(*args: list[object], **kwargs: dict[object, object]) -> Any: # type: ignore[explicit-any] + """ + Wrapper function around a given/decorated function, which cares about measurement and storing the result + + :param args: Arguments for the original function + :param kwargs: Keyword arguments for the original function + """ + if not EXECUTE_TIME_MEASUREMENTS: + return func(*args, **kwargs) + + start = timer() + # Execute original function + result = func(*args, **kwargs) + end = timer() + + runtime = end - start + + mt_name = func.__name__ if name is None else name + + mt_id = f"{category}_{func.__name__}" + + if mt_id not in TIME_MEASUREMENTS: + TIME_MEASUREMENTS[mt_id] = { + "name": mt_name, + "category": category, + "source": source, + "doc": func.__doc__, + "file": inspect.getfile(func), + "line": inspect.getsourcelines(func)[1], + "amount": 0, + "overall": 0, + "avg": None, + "min": None, + "max": None, + "min_max_spread": None, + "max_params": {"args": [], "kwargs": {}}, + } + + runtime_dict = TIME_MEASUREMENTS[mt_id] + + runtime_dict["amount"] += 1 + runtime_dict["overall"] += runtime + + if runtime_dict["min"] is None or runtime < runtime_dict["min"]: + runtime_dict["min"] = runtime + + if runtime_dict["max"] is None or runtime > runtime_dict["max"]: + runtime_dict["max"] = runtime + runtime_dict["max_params"] = { # Store parameters as a shorten string + "args": str([str(arg)[:80] for arg in args]), + "kwargs": str( + {key: str(value)[:80] for key, value in kwargs.items()} + ), + } + runtime_dict["min_max_spread"] = ( + runtime_dict["max"] / runtime_dict["min"] * 100 + ) + runtime_dict["avg"] = runtime_dict["overall"] / runtime_dict["amount"] + return result + + return wrapper # type: ignore[return-value] + + return inner + + +def measure_time_func( # type: ignore[explicit-any] + func: T, + category: str | None = None, + source: str = "internal", + name: str | None = None, +) -> T: + """Wrapper for measuring the needed execution time of a specific function. + + Usage as function:: + + from sphinx_needs.utils import measure_time + + # Old call: my_cool_function(a,b,c) + new_func = measure_time_func('my_category', func=my_cool_function) + new_func(a,b,c) + """ + return measure_time(category, source, name)(func) + + +def _print_timing_results() -> None: + for value in TIME_MEASUREMENTS.values(): + print(value["name"]) + print(f" amount: {value['amount']}") + print(f" overall: {value['overall']:2f}") + print(f" avg: {value['avg']:2f}") + print(f" max: {value['max']:2f}") + print(f" min: {value['min']:2f} \n") + + +def _store_timing_results_json(app: Sphinx, build_data: dict[str, Any]) -> None: # type: ignore[explicit-any] + json_result_path = Path(app.outdir) / "debug_measurement.json" + + data = {"build": build_data, "measurements": TIME_MEASUREMENTS} + with json_result_path.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=4) + print(f"Timing measurement results (JSON) stored under {json_result_path}") + + +def _store_timing_results_html(app: Sphinx, build_data: dict[str, Any]) -> None: # type: ignore[explicit-any] + jinja_env = Environment( + loader=PackageLoader("sphinx_needs"), autoescape=select_autoescape() + ) + template = jinja_env.get_template("time_measurements.html") + out_file = Path(str(app.outdir)) / "debug_measurement.html" + with out_file.open("w", encoding="utf-8") as f: + f.write(template.render(data=TIME_MEASUREMENTS, build_data=build_data)) + print(f"Timing measurement report (HTML) stored under {out_file}") + + +def process_timing(app: Sphinx, _exception: Exception | None) -> None: + if EXECUTE_TIME_MEASUREMENTS: + build_data = { + "project": app.config["project"], + "start": START_TIME, + "end": timer(), + "duration": timer() - START_TIME, + "timestamp": datetime.now().isoformat(), + } + + _print_timing_results() + _store_timing_results_json(app, build_data) + _store_timing_results_html(app, build_data) diff --git a/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py new file mode 100644 index 0000000..1a5b2db --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/directives/src_trace.py @@ -0,0 +1,340 @@ +from collections.abc import Callable +from pathlib import Path +import subprocess +from typing import Any, ClassVar, cast + +from docutils import nodes +from docutils.parsers.rst import directives +from packaging.version import Version +import sphinx +from sphinx.util.docutils import SphinxDirective +from sphinx_needs.api import add_need # type: ignore[import-untyped] +from sphinx_needs.utils import add_doc # type: ignore[import-untyped] + +from sphinx_codelinks.source_discovery.source_discover import SourceDiscover +from sphinx_codelinks.sphinx_extension.config import ( + SRC_TRACE_CACHE, + SrcTraceProjectConfigType, + SrcTraceSphinxConfig, + file_lineno_href, +) +from sphinx_codelinks.sphinx_extension.debug import measure_time +from sphinx_codelinks.virtual_docs.ubt_models import UBTComment +from sphinx_codelinks.virtual_docs.utils import get_file_types +from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs + +sphinx_version = sphinx.__version__ + + +if Version(sphinx_version) >= Version("1.6"): + from sphinx.util import logging +else: + import logging # type: ignore[no-redef] + +logger = logging.getLogger(__name__) + + +def generate_str_link_name( + comment: UBTComment, target_filepath: Path, target_dir: str +) -> str: + if comment.start_line == comment.end_line: + lineno = f"L{comment.start_line}" + else: + lineno = f"L{comment.start_line}-L{comment.end_line}" + url = str(target_filepath.relative_to(target_dir)) + f"#{lineno}" + + return url + + +def get_git_commit_id(src_dir: Path) -> str: + try: + commit_id = ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=src_dir) # noqa: S607 + .decode("utf-8") + .strip() + ) + except subprocess.CalledProcessError as err: + # raise RuntimeError("Failed to get the latest commit ID") from err + logger.warning(f"Failed to get the latest commit ID: {err}") + commit_id = "" + return commit_id + + +def get_git_root(cwd: Path = Path()) -> Path | None: + try: + # Run the git command to get the root directory + git_root = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], # noqa: S607 + cwd=cwd, + text=True, # Ensures the output is a string + ).strip() + return Path(git_root) + except subprocess.CalledProcessError: + logger.warning(f"Failed to get the Git root directory for {cwd}.") + return None + + +def validate_option(options: dict[str, str]) -> None: + if "project" not in options: + raise ValueError("Project option must be set.") + if "file" in options and "directory" in options: + raise ValueError("Either file or directory options can be set.") + + +class SourceTracing(nodes.General, nodes.Element): + pass + + +class SourceTracingDirective(SphinxDirective): + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = True + # this enables content in the directive + has_content = True + option_spec: ClassVar[dict[str, Callable[[str], str]] | None] = { + "id": directives.unchanged_required, + "project": directives.unchanged_required, + "file": directives.unchanged_required, + "directory": directives.unchanged_required, + } + + @measure_time("src-trace") + def run(self) -> list[nodes.Node]: + validate_option(self.options) + + project = self.options["project"] + title = self.arguments[0] + # get source tracing config + src_trace_sphinx_config = SrcTraceSphinxConfig(self.env.config) + + # load config + src_trace_conf: SrcTraceProjectConfigType = src_trace_sphinx_config.projects[ + project + ] + comment_type = src_trace_conf["comment_type"] + oneline_comment_style = src_trace_conf["oneline_comment_style"] + + src_dir = self.locate_src_dir(src_trace_sphinx_config, src_trace_conf) + + out_dir = Path(self.env.app.outdir) + # the directory where the source files are copied to + target_dir = out_dir / src_dir.name + + extra_options = {"project": project} + source_files = self.get_src_files(self.options, src_dir, src_trace_conf) + + # add source files into the dependency + # https://www.sphinx-doc.org/en/master/extdev/envapi.html#sphinx.environment.BuildEnvironment.note_dependency + for source_file in source_files: + self.env.note_dependency(str(source_file.resolve())) + + virtual_docs = VirtualDocs( + source_files, + str(src_dir), + str(out_dir / SRC_TRACE_CACHE), + oneline_comment_style, + comment_type=comment_type, + ) + virtual_docs.collect() + + needs = [] + + # create the need for src-trace directive + src_trace_need = add_need( + app=self.env.app, # The Sphinx application object + state=self.state, # The docutils state object + docname=self.env.docname, # The current document name + lineno=self.lineno, # The line number where the directive is used + need_type="srctrace", # The type of the need + title=title, # The title of the need + **extra_options, + ) + needs.extend(src_trace_need) + + # inject needs_string_links config before add_need() + # https://sphinx-needs.readthedocs.io/en/latest/configuration.html#needs-string-links + # local URL + local_url_field = None + remote_url_field = None + if src_trace_sphinx_config.set_local_url: + local_url_field = src_trace_sphinx_config.local_url_field + self.env.config.needs_string_links[local_url_field] = { + "regex": r"^(?P.+?)\.[^\.]+#L(?P\d+)", + "link_url": ( + f"file://{target_dir!s}/{{{{value}}}}.html#L-{{{{lineno}}}}" + ), + "link_name": "{{value}}#L{{lineno}}", + "options": [local_url_field], + } + if ( + src_trace_sphinx_config.set_remote_url + and src_trace_conf["remote_url_pattern"] + ): + git_root_path: Path | None = get_git_root(src_dir) + remote_url_field = src_trace_sphinx_config.remote_url_field + commit_id = get_git_commit_id(src_dir) + if git_root_path is None: + # No git root found, use the source directory as the remote source directory + remote_src_dir = src_dir + else: + remote_src_dir = src_dir.relative_to(git_root_path) + remote_url_pattern = src_trace_conf["remote_url_pattern"].format( + commit=commit_id, + path=f"{remote_src_dir}/" + "{{value}}", + line="{{lineno}}", + ) + self.env.config.needs_string_links[remote_url_field] = { + "regex": r"^(?P.+)#L(?P.*)?", + "link_url": remote_url_pattern, + "link_name": "{{value}}#L{{lineno}}", + "options": [remote_url_field], + } + dirs = { + "src_dir": src_dir, + "out_dir": out_dir, + "target_dir": target_dir, + } + # render needs from the source files + rendered_needs = self.render_needs( + virtual_docs, + local_url_field, + remote_url_field, + dirs, + ) + if rendered_needs: + needs.extend(rendered_needs) + + # virtual docs caching + virtual_docs.dump_virtual_docs() + virtual_docs.cache.update_cache() + + # for post-processing of need links + # https://github.com/useblocks/sphinx-needs/issues/1210 + add_doc(self.env, self.env.docname) + + return needs + + def get_src_files( + self, + extra_options: dict[str, str], + src_dir: Path, + src_trace_conf: SrcTraceProjectConfigType, + ) -> list[Path]: + source_files = [] + if "file" in self.options: + file: str = self.options["file"] + filepath = src_dir / file + source_files.append(filepath.resolve()) + extra_options["file"] = file + else: + directory = self.options.get("directory") + if directory is None: + # when neither "file" and "directory" are given, the project root dir is by default + directory = "./" + else: + extra_options["directory"] = directory + dir_path = src_dir / directory + file_types = get_file_types(src_trace_conf["comment_type"]) + source_discover = SourceDiscover( + dir_path, + gitignore=src_trace_conf["gitignore"], + include=src_trace_conf["include"], + exclude=src_trace_conf["exclude"], + file_types=file_types, + ) + source_files.extend(source_discover.source_paths) + + return source_files + + def locate_src_dir( + self, + src_trace_sphinx_config: SrcTraceSphinxConfig, + src_trace_conf: SrcTraceProjectConfigType, + ) -> Path: + """Locate the source directory based on the configuration.""" + # src dir in src_trace_conf is relative to conf_dir by default + conf_dir = Path(self.env.app.confdir) + # if config toml file is used, src dir is relative to the config toml + if src_trace_sphinx_config.config_from_toml: + src_trace_toml_path = Path(src_trace_sphinx_config.config_from_toml) + conf_dir = conf_dir / src_trace_toml_path.parent + + src_dir = (conf_dir / src_trace_conf["src_dir"]).resolve() + return src_dir + + def render_needs( + self, + virtual_docs: VirtualDocs, + local_url_field: str | None, + remote_url_field: str | None, + dirs: dict[str, Path], + ) -> list[nodes.Node]: + """Render the needs from the virtual docs""" + rendered_needs: list[nodes.Node] = [] + for virtual_doc in virtual_docs.virtual_docs: + # # add source files into the dependency + # # https://www.sphinx-doc.org/en/master/extdev/envapi.html#sphinx.environment.BuildEnvironment.note_dependency + # self.env.note_dependency(str(virtual_doc.filepath.resolve())) + + filepath = virtual_doc.filepath + target_filepath = dirs["target_dir"] / filepath.relative_to(dirs["src_dir"]) + # mapping between lineno and need link in docs for local url + lineno_href = {} + # The link to the documentation page for the source file + docs_href = f"{dirs['out_dir'] / self.env.docname}.html" + if local_url_field: + # copy files to _build/html + target_filepath.parent.mkdir(parents=True, exist_ok=True) + target_filepath.write_text(filepath.read_text()) + for comment in virtual_doc.comments: + # Always generate link_name to avoid unbound errors + link_name = None + if local_url_field or remote_url_field: + # generate link name + link_name = generate_str_link_name( + comment, target_filepath, str(dirs["target_dir"]) + ) + if comment.resolved_marker: + # render needs from one-line marker + kwargs: dict[str, str | list[str]] = { + field_name: field_value + for field_name, field_value in comment.resolved_marker.items() + if field_name + not in [ + "title", + "type", + ] # title and type are mandatory for add_need() + } + + if local_url_field and link_name is not None: + kwargs[local_url_field] = link_name + if remote_url_field and link_name is not None: + kwargs[remote_url_field] = link_name + + marker_needs: list[nodes.Node] = add_need( + app=self.env.app, # The Sphinx application object + state=self.state, # The docutils state object + docname=self.env.docname, # The current document name + lineno=self.lineno, # The line number where the directive is used + need_type=str( + comment.resolved_marker["type"] + ), # The type of the need + title=str( + comment.resolved_marker["title"] + ), # The title of the need + **cast(dict[str, Any], kwargs), # type: ignore[explicit-any] + ) + rendered_needs.extend(marker_needs) + if local_url_field: + # save the mapping of need links and line numbers of source codes + # for the later use in `html-collect-pages` + lineno_href[comment.start_line] = ( + f"{docs_href}#{comment.resolved_marker['id']}" + ) + + if local_url_field: + # save the mappings of need links and line numbers of source codes + # for the later use in `html-collect-pages` + file_lineno_href.mappings[str(target_filepath)] = lineno_href + + return rendered_needs diff --git a/src/sphinx_codelinks/sphinx_extension/html_wrapper.py b/src/sphinx_codelinks/sphinx_extension/html_wrapper.py new file mode 100644 index 0000000..ca85822 --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/html_wrapper.py @@ -0,0 +1,52 @@ +from collections.abc import Generator +from pathlib import Path +from typing import Any + +from pygments import highlight +from pygments.formatters import HtmlFormatter +from pygments.lexers import CLexer + + +class LineFormatter(HtmlFormatter): # type: ignore[type-arg] + def __init__(self, lineno_href: dict[int, str], *args: Any, **kwargs: Any) -> None: # type: ignore[explicit-any] + super().__init__(*args, **kwargs) + self.lineno_href = lineno_href + + def wrap(self, source: Generator[Any]) -> Generator[Any]: # type: ignore[explicit-any] + return self._wrap_custom_lines(super().wrap(source)) # type: ignore[no-untyped-call] + + def _wrap_custom_lines(self, source: Generator[Any]) -> Generator[Any]: # type: ignore[explicit-any] + lineno = 0 + for is_line, line_html in source: + if is_line: + lineno += 1 + if lineno in self.lineno_href: + lineno_achor, inline_lineno, code_span = line_html.split("") + # Ensure the anchor is closed + inline_lineno = inline_lineno + "" + lineno_achor = lineno_achor + "" + # make the code as a link to the documentation + yield ( + is_line, + f'[docs]{line_html}', + ) + else: + yield is_line, f"{line_html}" + else: + yield is_line, line_html + + +def html_wrapper(filepath: Path, lineno_href: dict[int, str]) -> str: + code = filepath.read_text() + + formatter = LineFormatter( + lineno_href=lineno_href, + # use inline, as table may make lineno and code misaligned with certain Sphinx themes + linenos="inline", + lineanchors="L", # Adds anchor IDs like id="L-20" + anchorlinenos=True, # Makes line numbers clickable (link to #L-20) + wrapcode=False, # Wraps the code in a
with class "highlight" + ) + + html_content: str = highlight(code, CLexer(stripnl=False), formatter) + return html_content diff --git a/src/sphinx_codelinks/sphinx_extension/source_tracing.py b/src/sphinx_codelinks/sphinx_extension/source_tracing.py new file mode 100644 index 0000000..21b7cf8 --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/source_tracing.py @@ -0,0 +1,222 @@ +from collections.abc import Iterator # only in python 3.11 afterwards +import contextlib +from pathlib import Path +from timeit import default_timer as timer # Used for timing measurements +import tomllib +from typing import Any, cast + +from sphinx.application import Sphinx +from sphinx.config import Config as _SphinxConfig +from sphinx.environment import BuildEnvironment +from sphinx.util import logging +from sphinx.util.fileutil import copy_asset +from sphinx_needs.api import ( # type: ignore[import-untyped] + add_extra_option, + add_need_type, +) + +from sphinx_codelinks.sphinx_extension import debug +from sphinx_codelinks.sphinx_extension.config import ( + SRC_TRACE_CACHE, + SrcTraceConfigType, + SrcTraceProjectConfigType, + SrcTraceSphinxConfig, + check_configuration, + file_lineno_href, +) +from sphinx_codelinks.sphinx_extension.directives.src_trace import ( + SourceTracing, + SourceTracingDirective, +) +from sphinx_codelinks.sphinx_extension.html_wrapper import html_wrapper +from sphinx_codelinks.virtual_docs.config import ( + OneLineCommentStyle, + OneLineCommentStyleType, +) +from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs + +logger = logging.getLogger(__name__) + + +def setup(app: Sphinx) -> dict[str, Any]: # type: ignore[explicit-any] + app.add_node(SourceTracing) + app.add_directive("src-trace", SourceTracingDirective) + SrcTraceSphinxConfig.add_config_values(app) + + app.connect("config-inited", load_config_from_toml, priority=10) + app.connect( + "config-inited", update_sn_extra_options, priority=11 + ) # run early otherwise, extra options are not set for nested_parse + app.connect("config-inited", update_sn_types) + app.connect("config-inited", check_sphinx_configuration) + + app.connect("env-before-read-docs", prepare_env) + app.connect("html-collect-pages", generate_code_page) + app.connect("html-page-context", add_custom_css) + app.connect("builder-inited", builder_inited) + app.connect("build-finished", emit_warnings) + app.connect("build-finished", debug.process_timing) + return { + "version": "builtin", + "parallel_read_safe": True, + "parallel_write_safe": True, + } + + +def builder_inited(app: Sphinx) -> None: + custom_css = Path(__file__).parent / "ub_sct.css" + copy_asset(custom_css, Path(app.outdir) / "_static" / "source_tracing") + + +def add_custom_css( # type: ignore[explicit-any] + app: Sphinx, + pagename: str, + templatename: str, + context: dict[str, Any], + _doctree: Any, +) -> None: + target_htmls = { + str(Path(file_path).relative_to(app.outdir).with_suffix("")) + for file_path in file_lineno_href.mappings + } + + if pagename in target_htmls and templatename == "page.html": + if "css_files" not in context: + context["css_files"] = [] + context["css_files"].append( + "_static/source_tracing/ub_sct.css" + ) # Add the custom CSS file to the context + + +def generate_code_page( + app: Sphinx, +) -> Iterator[tuple[str, dict[str, str], str]] | None: + for file, lineno_href in file_lineno_href.mappings.items(): + file_path = Path(file) + pagename = str((file_path.relative_to(app.outdir)).with_suffix("")) + + html_content = html_wrapper( + file_path, + lineno_href=lineno_href, + ) + + context = { + "title": f"Source Code Tracing: {file_path.name}", + "body": html_content, + } + + yield pagename, context, "page.html" + + file_lineno_href.mappings.clear() # Clear the mappings after generating the pages + return None + + +def load_config_from_toml(app: Sphinx, config: _SphinxConfig) -> None: + """Load the configuration from a TOML file, if defined in conf.py.""" + src_trc_sphinx_config = SrcTraceSphinxConfig(config) + if src_trc_sphinx_config.config_from_toml is None: + return + + # resolve relative to confdir + toml_file = Path(app.confdir, src_trc_sphinx_config.config_from_toml).resolve() + # toml_path = src_trc_sphinx_config.from_toml_table + + if not toml_file.exists(): + logger.warning( + f"Source tracing configuration file {toml_file} does not exist. Using configuration from conf.py." + ) + return + try: + with toml_file.open("rb") as f: + toml_data = tomllib.load(f) + toml_data = toml_data["src_trace"] + if not isinstance(toml_data, dict): + raise Exception(f"data must be a dict in {toml_file}") + + except Exception as e: + logger.warning( + f"Failed to load source tracing configuration from {toml_file}: {e}" + ) + return + + set_config_to_sphinx( + src_trace_config=cast(SrcTraceConfigType, toml_data), config=config + ) + + +def set_config_to_sphinx( + src_trace_config: SrcTraceConfigType, config: _SphinxConfig +) -> None: + allowed_keys = SrcTraceSphinxConfig.field_names() + for key, value in src_trace_config.items(): + if key not in allowed_keys: + continue + if key == "projects": + for project_config in cast( + dict[str, SrcTraceProjectConfigType], value + ).values(): + oneline_comment_style: OneLineCommentStyleType | None = cast( + OneLineCommentStyleType, project_config.get("oneline_comment_style") + ) + if oneline_comment_style: + project_config["oneline_comment_style"] = OneLineCommentStyle( + **cast( + OneLineCommentStyleType, + project_config["oneline_comment_style"], + ) + ) + + config[f"src_trace_{key}"] = value + + +def update_sn_extra_options(app: Sphinx, config: _SphinxConfig) -> None: + src_trace_sphinx_config = SrcTraceSphinxConfig(config) + add_extra_option(app, "project") + add_extra_option(app, "file") + add_extra_option(app, "directory") + if src_trace_sphinx_config.set_local_url: + add_extra_option(app, src_trace_sphinx_config.local_url_field) + if src_trace_sphinx_config.set_remote_url: + add_extra_option(app, src_trace_sphinx_config.remote_url_field) + + +def update_sn_types(app: Sphinx, _config: _SphinxConfig) -> None: + add_need_type(app, "srctrace", "Src-Trace", "ST_", "#ffffff", "node") + + +def prepare_env(app: Sphinx, env: BuildEnvironment, _docnames: list[str]) -> None: # noqa: ARG001 # required by Sphinx + """ + Prepares the sphinx environment to store stc-trace internal data. + """ + src_trace_sphinx_config = SrcTraceSphinxConfig(app.config) + + # Set time measurement flag + if src_trace_sphinx_config.debug_measurement: + debug.START_TIME = timer() # Store the rough start time of Sphinx build + debug.EXECUTE_TIME_MEASUREMENTS = True + + if src_trace_sphinx_config.debug_filters: + with contextlib.suppress(FileNotFoundError): + Path(str(app.outdir), "debug_filters.jsonl").unlink() + + +def check_sphinx_configuration(app: Sphinx, _config: _SphinxConfig) -> None: + config = SrcTraceSphinxConfig(app.config) + errors = check_configuration(config) + if errors: + raise Exception("\n".join(errors)) + + +def emit_warnings( + app: Sphinx, + _env: BuildEnvironment, +) -> None: + warnings = VirtualDocs.load_warnings(Path(app.outdir) / SRC_TRACE_CACHE) + if not warnings: + return + for warning in warnings: + logger.warning( + f"{warning.file_path}:{warning.lineno}: {warning.msg}", + type=warning.type, + subtype=warning.sub_type, + ) diff --git a/src/sphinx_codelinks/sphinx_extension/ub_sct.css b/src/sphinx_codelinks/sphinx_extension/ub_sct.css new file mode 100644 index 0000000..8e6941d --- /dev/null +++ b/src/sphinx_codelinks/sphinx_extension/ub_sct.css @@ -0,0 +1,8 @@ +.highlight { + position:relative +} + +.viewcode-back{ + position: absolute; + right:0; +} diff --git a/src/sphinx_codelinks/virtual_docs/config.py b/src/sphinx_codelinks/virtual_docs/config.py new file mode 100644 index 0000000..6a4fbfa --- /dev/null +++ b/src/sphinx_codelinks/virtual_docs/config.py @@ -0,0 +1,256 @@ +from dataclasses import MISSING, dataclass, field, fields +import logging +import os +from pathlib import Path +from typing import Any, Literal, TypedDict, cast + +from jsonschema import ValidationError, validate + +# initialize logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# log to the console +console = logging.StreamHandler() +console.setLevel(logging.INFO) +logger.addHandler(console) + +ESCAPE = "\\" +SUPPORTED_COMMENT_TYPES = {"c", "h", "cpp", "hpp"} + + +class VirtualDocsConfigType(TypedDict): + src_files: list[Path] | None + src_dir: Path + output_dir: Path + comment_type: str + + +@dataclass +class VirtualDocsConfig: + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + src_files: list[Path] = field( + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """A list of source files to be processed.""" + + src_dir: Path = field( + default_factory=lambda: Path.cwd(), metadata={"schema": {"type": "string"}} + ) + """The root of the source directory.""" + + output_dir: Path = field( + default=Path("output"), metadata={"schema": {"type": "string"}} + ) + """The directory where the virtual documents and their caches will be stored.""" + + comment_type: str = field(default="c", metadata={"schema": {"type": "string"}}) + """The type of comment to be processed.""" + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[explicit-any] + return None + + def check_schema(self) -> list[str]: + errors = [] + for _field_name in self.field_names(): + schema = self.get_schema(_field_name) + value = getattr(self, _field_name) + if _field_name == "src_files": # adapt to json schema restriction + if isinstance(value, list): + value: list[str] = [str(src_file) for src_file in value] # type: ignore[no-redef] # only for value adaptation + elif isinstance(value, Path): # adapt to json schema restriction + value = str(value) + + try: + validate(instance=value, schema=schema) # type: ignore[arg-type] # validate has no type specified + except ValidationError as e: + errors.append( + f"Schema validation error in field '{_field_name}': {e.message}" + ) + return errors + + +class FieldConfig(TypedDict, total=False): + name: str + type: Literal["str", "list[str]"] + default: str | list[str] | None + + +class OneLineCommentStyleType(TypedDict): + start_sequence: str + end_sequence: str + field_split_char: str + needs_fields: list[FieldConfig] + + +@dataclass +class OneLineCommentStyle: + def __setattr__(self, name: str, value: Any) -> None: # type: ignore[explicit-any] + if name == "needs_fields": + # apply default to fields + self.apply_needs_field_default(value) + return super().__setattr__(name, value) + + @classmethod + def field_names(cls) -> set[str]: + return {item.name for item in fields(cls)} + + start_sequence: str = field(default="@", metadata={"schema": {"type": "string"}}) + """Chars sequence to indicate the start of the one-line comment.""" + + end_sequence: str = field( + default=os.linesep, metadata={"schema": {"type": "string"}} + ) + """Chars sequence to indicate the end of the one-line comment.""" + + field_split_char: str = field(default=",", metadata={"schema": {"type": "string"}}) + """Char sequence to split the fields.""" + + needs_fields: list[FieldConfig] = field( + default_factory=lambda: [ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + metadata={ + "required_fields": ["title", "type"], + "field_default": { + "type": "str", + }, + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "type": { + "type": "string", + "enum": ["str", "list[str]"], + "default": "str", + }, + "default": { + "anyOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}}, + ] + }, + }, + "required": ["name"], + "additionalProperties": False, + "allOf": [ + { + "if": {"properties": {"type": {"const": "list[str]"}}}, + "then": { + "properties": { + "default": { + "type": "array", + "items": {"type": "string"}, + } + } + }, + }, + { + "if": {"properties": {"type": {"const": "str"}}}, + "then": {"properties": {"default": {"type": "string"}}}, + }, + ], + }, + }, + }, + ) + + @classmethod + def apply_needs_field_default(cls, given_fields: list[FieldConfig]) -> None: + field_default = next( + _field.metadata["field_default"] + for _field in fields(cls) + if _field.name == "needs_fields" + ) + + for _field in given_fields: + for _default in field_default: + if _default not in _field: + _field[_default] = field_default[_default] # type: ignore[literal-required] # dynamically assign keys + + @classmethod + def get_required_fields(cls, name: str) -> list[str] | None: + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING: + return cast(list[str], _field.metadata["required_fields"]) + return None + + @classmethod + def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] + _field = next(_field for _field in fields(cls) if _field.name is name) + if _field.metadata is not MISSING and "schema" in _field.metadata: + return cast(dict[str, Any], _field.metadata["schema"]) # type: ignore[explicit-any] + return None + + def check_schema(self) -> list[str]: + errors = [] + for _field_name in self.field_names(): + schema = self.get_schema(_field_name) + value = getattr(self, _field_name) + try: + validate(instance=value, schema=schema) # type: ignore[arg-type] # validate has no type specified + except ValidationError as e: + if _field_name == "needs_fields": + need_field_name = value[e.path[0]]["name"] + errors.append( + f"Schema validation error in need_fields '{need_field_name}': {e.message}" + ) + else: + errors.append( + f"Schema validation error in field '{_field_name}': {e.message}" + ) + return errors + + def check_required_fields(self) -> list[str]: + errors = [] + required_fields = self.get_required_fields("needs_fields") + if required_fields is None: + errors.append("No required fields specified.") + return errors + given_field_names = [_field["name"] for _field in self.needs_fields] + missing_fields = set(required_fields) - set(given_field_names) + if len(missing_fields) != 0: + errors.append(f"Missing required fields: {sorted(missing_fields)}") + + return errors + + def check_fields_mutually_exclusive(self) -> list[str]: + errors = [] + needs_field_names = set() + for _field in self.needs_fields: + if _field["name"] in needs_field_names: + errors.append(f"Field '{_field['name']}' is defined multiple times.") + needs_field_names.add(_field["name"]) + return errors + + def check_fields_configuration(self) -> list[str]: + return ( + self.check_schema() + + self.check_required_fields() + + self.check_fields_mutually_exclusive() + ) + + def get_cnt_required_fields(self) -> int: + cnt_required_fields = 0 + for _field in self.needs_fields: + if _field.get("default") is None: + cnt_required_fields += 1 + return cnt_required_fields + + def get_pos_list_str(self) -> list[int]: + pos_list_str = [] + for idx, _field in enumerate(self.needs_fields): + if _field["type"] == "list[str]": + pos_list_str.append(idx + 1) + return pos_list_str diff --git a/src/sphinx_codelinks/virtual_docs/ubt_models.py b/src/sphinx_codelinks/virtual_docs/ubt_models.py new file mode 100644 index 0000000..9bd2c89 --- /dev/null +++ b/src/sphinx_codelinks/virtual_docs/ubt_models.py @@ -0,0 +1,131 @@ +import json +from pathlib import Path +from typing import cast + + +class MultipleMarkerError(Exception): + """Custom exception for multiple markers in a comment.""" + + +class UBTComment: + """Wrap Comment object from comment_parser.""" + + def __init__( + self, + text: str, + start_line: int, + resolved_marker: dict[str, str | list[str]], + marker_type: str = "oneline", + ): + self.text = text + # start and end columns are not supported by comment_parser + # start_line and end_line are the line number of the comment by default. + # If the marked text exists, they will be line numbers of that. + self.start_line = start_line + # so far only one-line comment is taken. end_line is kept for the future multi-line styles + self.end_line = ( + self.start_line + self.text.count("\n") - 1 + if self.text.count("\n") + else self.start_line + ) + self.resolved_marker: dict[str, str | list[str]] = resolved_marker + self.marker_type: str = marker_type + + def __eq__(self, value): + if isinstance(value, UBTComment): + return self.__dict__ == value.__dict__ + return False + + def __hash__(self) -> int: + return hash(self.__dict__) + + def to_dict(self) -> dict[str, dict[str, str | list[str]] | str | int]: + return { + "text": self.text, + "start_line": self.start_line, + "end_line": self.end_line, + "marker_type": self.marker_type, + "resolved_marker": self.resolved_marker, + } + + +class UBTSourceFile: + def __init__( + self, + filepath: Path, + src_dir: Path, + comments: list[UBTComment] | None = None, + output_dir: str = "./", + ): + self.filepath: Path = filepath + self.src_dir: Path = src_dir + self.comments: list[UBTComment] = [] + if comments: + self.comments.extend(comments) + self.output_dir = Path(output_dir) + self.changed_date = self.filepath.stat().st_mtime + + def __eq__(self, value): + if isinstance(value, UBTSourceFile): + return self.__dict__ == value.__dict__ + return False + + def __hash__(self) -> int: + return hash(self.__dict__) + + def add_comment(self, comment: UBTComment) -> None: + self.comments.append(comment) + + def add_comments(self, comment: list[UBTComment]) -> None: + self.comments.extend(comment) + + def to_json(self) -> None: + comments = [comment.__dict__ for comment in self.comments] + output_path = self.output_dir / self.filepath.with_suffix(".json").relative_to( + self.src_dir + ) + if not output_path.parent.exists(): + output_path.parent.mkdir(parents=True) + with output_path.open("w") as f: + json.dump(comments, f) + + +class UBTCache: + def __init__( + self, + cache_path: str = "./ubt_cache.json", + uncached_files: list[UBTSourceFile] | None = None, + ): + if uncached_files is None: + uncached_files = [] + self.cache_path = Path(cache_path) + self.uncached_files = uncached_files + self.cached_files = self.load_cache() + + def load_cache(self) -> dict[str, float]: + if not self.cache_path.exists(): + return {} + with self.cache_path.open("r") as f: + cached_files = cast(dict[str, float], json.load(f)) + return cached_files + + def add_uncached_files(self, uncached_files: list[UBTSourceFile]) -> None: + self.uncached_files.extend(uncached_files) + for uncached_file in uncached_files: + if str(uncached_file.filepath) in self.cached_files: + self.cached_files.pop(str(uncached_file.filepath)) + + def update_cache(self) -> None: + for src_file in self.uncached_files: + if ( + str(src_file.filepath) in self.cached_files + and src_file.changed_date == self.cached_files[str(src_file.filepath)] + ): + continue + self.cached_files[str(src_file.filepath)] = src_file.changed_date + # remove cached files from uncached_files + self.uncached_files = [] + if not self.cache_path.parent.exists(): + self.cache_path.parent.mkdir(parents=True) + with self.cache_path.open("w") as f: + json.dump(self.cached_files, f) diff --git a/src/sphinx_codelinks/virtual_docs/utils.py b/src/sphinx_codelinks/virtual_docs/utils.py new file mode 100644 index 0000000..c30eda1 --- /dev/null +++ b/src/sphinx_codelinks/virtual_docs/utils.py @@ -0,0 +1,215 @@ +from dataclasses import dataclass +from enum import Enum +import logging +import os + +from sphinx_codelinks.virtual_docs.config import ( + ESCAPE, + SUPPORTED_COMMENT_TYPES, + OneLineCommentStyle, +) + +# initialize logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# log to the console +console = logging.StreamHandler() +console.setLevel(logging.INFO) +logger.addHandler(console) + + +class WarningSubTypeEnum(str, Enum): + """Enum for warning sub types.""" + + too_many_fields = "too_many_fields" + too_few_fields = "too_few_fields" + missing_square_brackets = "missing_square_brackets" + not_start_or_end_with_square_brackets = "not_start_or_end_with_square_brackets" + newline_in_field = "newline_in_field" + + +@dataclass +class OnelineParserInvalidWarning: + """Invalid oneline comments.""" + + sub_type: WarningSubTypeEnum + msg: str + + +def oneline_parser( # noqa: PLR0912, PLR0911 # handel warnings + oneline: str, oneline_config: OneLineCommentStyle +) -> dict[str, str | list[str]] | OnelineParserInvalidWarning | None: + """ + Extract the string from the custom one-line comment style with the following steps. + + - Locate the start and end sequences + - extract the string between them + - apply custom_split to split the strings into a list of fields by `field_split_char` + - check the number of required fields and the max number of the given fields + - split the strings located in the field with `type: list[str]` to a list of string + - introduce the default values to those fields which are not given + """ + # find indices start and end char + start_idx = oneline.find(oneline_config.start_sequence) + end_idx = oneline.rfind(oneline_config.end_sequence) + if start_idx == -1 or end_idx == -1: + # start or end sequences do not exist + return None + + # extract the string wrapped by start and end + string = oneline[start_idx + len(oneline_config.start_sequence) : end_idx] + + # numbers of needs_fields which are required + cnt_required_fields = oneline_config.get_cnt_required_fields() + # indices of the field which has type:list[str] + positions_list_str = oneline_config.get_pos_list_str() + + min_fields = cnt_required_fields + max_fields = len(oneline_config.needs_fields) + + string_fields = [ + _field.strip(" ") + for _field in custom_split( + string, oneline_config.field_split_char, positions_list_str + ) + ] + if len(string_fields) < min_fields: + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.too_few_fields, + msg=f"{len(string_fields)} given fields. They shall be more than {min_fields}", + ) + + if len(string_fields) > max_fields: + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.too_many_fields, + msg=f"{len(string_fields)} given fields. They shall be less than {max_fields}", + ) + resolved: dict[str, str | list[str]] = {} + for idx in range(len(oneline_config.needs_fields)): + field_name: str = oneline_config.needs_fields[idx]["name"] + if len(string_fields) > idx: + # given fields + if is_newline_in_field(string_fields[idx]): + # the case where the field contains a new line character + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.newline_in_field, + msg=f"Field {field_name} has newline character. It is not allowed", + ) + if oneline_config.needs_fields[idx]["type"] == "str": + resolved[field_name] = string_fields[idx] + elif oneline_config.needs_fields[idx]["type"] == "list[str]": + # find the indices of "[" and "]" + start_idx = string_fields[idx].find("[") + end_idx = string_fields[idx].rfind("]") + if start_idx == -1 or end_idx == -1: + # brackets are not found + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.missing_square_brackets, + msg=f"Field {field_name} with 'type': '{oneline_config.needs_fields[idx]['type']}' must be given with '[]' brackets", + ) + + if start_idx != 0 or end_idx != len(string_fields[idx]) - 1: + # brackets are found but not at the beginning and the end + return OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.not_start_or_end_with_square_brackets, + msg=f"Field {field_name} with 'type': '{oneline_config.needs_fields[idx]['type']}' must start with '[' and end with ']'", + ) + + string_items = string_fields[idx][start_idx + 1 : end_idx] + + if not string_items.strip(): + # the case where the empty string ("") or only spaces between "[" "]" + resolved[field_name] = [] + else: + items = [_item.strip() for _item in custom_split(string_items, ",")] + resolved[field_name] = [item.strip() for item in items] + else: + # for not given fields, introduce the default + default = oneline_config.needs_fields[idx].get("default") + if default is None: + continue + resolved[field_name] = default + + return resolved + + +def custom_split( + string: str, delimiter: str, positions_list_str: list[int] | None = None +) -> list[str]: + """ + A string shall be split with the following conditions: + + - To use special chars in literal , escape ('\') must be used + - String shall be split by the given delimiter + - In a field with `type: str`: + - Special chars are delimiter, '\', '[' and ']' + - In a field with `type: list[str]`: + - Special chars are only '[' and ']' + + When the string is given without any fields with `type: list[str]` (positions_list_str=None), + it's considered as it is in a field with `type: str`. + """ + if positions_list_str is None: + positions_list_str = [] + escape_chars = [delimiter, "[", "]", ESCAPE] + field = [] # a list of string for a field + fields: list[str] = [] # a list of string which contains + leading_escape = False + expect_closing_bracket = False + + for char in string: + # +1 to locate the current field position + current_field_idx = len(fields) + 1 + is_list_str_field = current_field_idx in positions_list_str + + if leading_escape: + if char not in escape_chars: + # leading escape is considered as a literal + field.append(ESCAPE) + field.append(char) + leading_escape = False + continue + + if char == ESCAPE and not is_list_str_field: + leading_escape = True + continue + + if char == delimiter: + if is_list_str_field and expect_closing_bracket: + # delimiter occurs in the field with type:list[str] + field.append(char) + else: + fields.append("".join(field)) + field = [] + continue + + if is_list_str_field: + if char == "[": + expect_closing_bracket = True + if char == "]": + expect_closing_bracket = False + + field.append(char) + + # add last field + fields.append("".join(field)) + return fields + + +def is_newline_in_field(field: str) -> bool: + """ + Check if the field contains a new line character. + """ + return os.linesep in field + + +def get_file_types(comment_type: str) -> list[str] | None: + """ + Get the list of file types to be discovered. + """ + file_types = ( + list(SUPPORTED_COMMENT_TYPES) + if comment_type in SUPPORTED_COMMENT_TYPES + else None + ) + return file_types diff --git a/src/sphinx_codelinks/virtual_docs/virtual_docs.py b/src/sphinx_codelinks/virtual_docs/virtual_docs.py new file mode 100644 index 0000000..62bb299 --- /dev/null +++ b/src/sphinx_codelinks/virtual_docs/virtual_docs.py @@ -0,0 +1,238 @@ +from dataclasses import dataclass +import json +import logging +import os +from pathlib import Path + +from comment_parser.parsers.c_parser import ( # type: ignore[import-untyped] + extract_comments, +) +from comment_parser.parsers.common import Comment # type: ignore[import-untyped] + +from sphinx_codelinks.virtual_docs.config import ( + SUPPORTED_COMMENT_TYPES, + OneLineCommentStyle, +) +from sphinx_codelinks.virtual_docs.ubt_models import UBTCache, UBTComment, UBTSourceFile +from sphinx_codelinks.virtual_docs.utils import ( + OnelineParserInvalidWarning, + oneline_parser, +) + +# initialize logger +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +# log to the console +console = logging.StreamHandler() +console.setLevel(logging.INFO) +logger.addHandler(console) + + +@dataclass +class VirtualDocsOneLineWarning: + file_path: str + lineno: int + msg: str + type: str + sub_type: str + + +class VirtualDocs: + warning_filepath: Path = Path("cached_warnings") / "vdocs_warnings.json" + + def __init__( + self, + src_files: list[Path], + src_dir: str, + output_dir: str, + oneline_comment_style: OneLineCommentStyle, + comment_type: str = "c", + ) -> None: + self.src_files = src_files + self.src_dir = Path(src_dir) + self.output_dir = Path(output_dir) + self.comment_type = comment_type + self.cache = UBTCache(str(self.output_dir / "ubt_cache.json")) + self.cache.add_uncached_files(self._uncached_files()) + self.virtual_docs: list[UBTSourceFile] = [] + self.oneline_comment_style = oneline_comment_style + self.oneline_warnings: list[VirtualDocsOneLineWarning] = [] + self.warnings_path = self.output_dir / VirtualDocs.warning_filepath + + def collect(self) -> None: + # import C parser to avoid `python-magic` dependency + # https://github.com/jeanralphaviles/comment_parser?tab=readme-ov-file#osx-and-windows + if self.comment_type not in SUPPORTED_COMMENT_TYPES: + raise Exception( + f"Unsupported comment type: {self.comment_type}. Supported types are: {SUPPORTED_COMMENT_TYPES}." + ) + + virtual_docs = [] + self.load_virtual_docs() + # parse all uncached files + for src_file in self.cache.uncached_files: + ml_comments: list[Comment] = [] + oneline_comments: list[Comment] = [] + with src_file.filepath.open("r", encoding="utf-8") as code: + comments = extract_comments(code.read()) + + # separate one-line and multi-line comments + for comment in comments: + if comment.is_multiline(): + ml_comments.append(comment) + else: + oneline_comments.append(comment) + + # break all multi-line comments to single-line comments + for comment in ml_comments: + single_lines = comment.text().splitlines() + for idx, line in enumerate(single_lines): + oneline_comments.append(Comment(line, comment.line_number() + idx)) + + ubt_comments: list[UBTComment] = [] + + for comment in oneline_comments: + resolved = oneline_parser( + f"{comment.text()}{os.linesep}", self.oneline_comment_style + ) + + if isinstance(resolved, OnelineParserInvalidWarning): + self.oneline_warnings.append( + VirtualDocsOneLineWarning( + str(src_file.filepath), + comment.line_number(), + resolved.msg, + type="oneline", + sub_type=resolved.sub_type.value, + ) + ) + continue + + if resolved: + ubt_comments.append( + UBTComment( + f"{comment.text()}{os.linesep}", + start_line=comment.line_number(), + resolved_marker=resolved, + ) + ) + + ubt_comments.sort(key=lambda x: x.start_line) + src_file.add_comments(ubt_comments) + virtual_docs.append(src_file) + + self.virtual_docs.extend(virtual_docs) + self.update_warnings() + + def _uncached_files(self) -> list[UBTSourceFile]: + uncached_files = [] + for src_file in self.src_files: + ubt_src_file = UBTSourceFile( + src_file, self.src_dir, output_dir=str(self.output_dir) + ) + # check cached virtual documents + if ( + str(src_file) in self.cache.cached_files + and ( + self.output_dir + / (src_file.with_suffix(".json").relative_to(self.src_dir)) + ).exists() + and self.cache.cached_files.get(str(ubt_src_file.filepath)) + == ubt_src_file.changed_date + ): + continue + uncached_files.append(ubt_src_file) + return uncached_files + + def dump_virtual_docs(self) -> None: + for src_file in self.virtual_docs: + src_file.to_json() + + @classmethod + def load_warnings( + cls, warnings_dir: Path + ) -> list[VirtualDocsOneLineWarning] | None: + """Load warnings from the given path. + + It mainly used for other apps or users to load warnings files directly. + """ + warnings_path = warnings_dir / cls.warning_filepath + if not warnings_path.exists(): + return None + with warnings_path.open("r") as f: + # load the json file and convert to VirtualDocsOneLineWarning] + warnings = json.load(f) + loaded_warnings = [ + VirtualDocsOneLineWarning(**warning) for warning in warnings + ] + return loaded_warnings + + def _load_warnings(self) -> list[VirtualDocsOneLineWarning] | None: + if not self.warnings_path.exists(): + return None + with self.warnings_path.open("r") as f: + # load the json file and convert to VirtualDocsOneLineWarning] + warnings = json.load(f) + loaded_warnings = [ + VirtualDocsOneLineWarning(**warning) for warning in warnings + ] + return loaded_warnings + + def update_warnings(self) -> None: + loaded_warnings = self._load_warnings() + current_warnings = [_warning.__dict__ for _warning in self.oneline_warnings] + if loaded_warnings: + _warnings = [_warning.__dict__ for _warning in loaded_warnings] + cached_warnings = [ + _warning + for _warning in _warnings + if not ( + _warning["file_path"] + in [str(src_file) for src_file in self.src_files] + and _warning["file_path"] + in [ + str(ubt_src_file.filepath) + for ubt_src_file in self.cache.uncached_files + ] + ) + ] + total_warning = cached_warnings + current_warnings + else: + total_warning = current_warnings + if not self.warnings_path.parent.exists(): + self.warnings_path.parent.mkdir(parents=True) + with self.warnings_path.open("w") as f: + json.dump( + total_warning, + f, + ) + + def load_virtual_docs(self) -> None: + # only load cached files that are in the self.src_files + src_files = [ + src_file + for src_file in self.cache.cached_files + if src_file in [str(file_path) for file_path in self.src_files] + ] + for src_file in src_files: + src_path = Path(src_file) + virt_doc_path = self.output_dir / ( + src_path.with_suffix(".json").relative_to(self.src_dir) + ) + if virt_doc_path.exists(): + ubt_src_file = UBTSourceFile( + src_path, self.src_dir, output_dir=str(self.output_dir) + ) + comments = json.load(virt_doc_path.open("r")) + ubt_src_file.add_comments( + [ + UBTComment( + text=comment["text"], + start_line=comment["start_line"], + resolved_marker=comment["resolved_marker"], + marker_type=comment["marker_type"], + ) + for comment in comments + ] + ) + self.virtual_docs.append(ubt_src_file) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project0-source_code0].doctree.xml b/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project0-source_code0].doctree.xml new file mode 100644 index 0000000..edf41b7 --- /dev/null +++ b/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project0-source_code0].doctree.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project1-source_code1].doctree.xml b/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project1-source_code1].doctree.xml new file mode 100644 index 0000000..ce9b94a --- /dev/null +++ b/tests/__snapshots__/test_src_trace/test_build_html[sphinx_project1-source_code1].doctree.xml @@ -0,0 +1,3 @@ + + + diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..dd9dd67 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,65 @@ +from pathlib import Path + +from docutils.nodes import document +import pytest +from syrupy.extensions.single_file import SingleFileSnapshotExtension, WriteMode + +pytest_plugins = "sphinx.testing.fixtures" + +TEST_DIR = Path(__file__).parent +SRC_TRACE_TOML = TEST_DIR / "data" / "sphinx" / "src_trace.toml" +BASIC_VDOC_TOML = TEST_DIR / "data" / "oneline_comment_basic" / "vdoc_config.toml" +DEFAULT_VDOC_TOML = TEST_DIR / "data" / "oneline_comment_default" / "vdoc_config.toml" +RECURSIVE_DIR_VDOC_TOML = TEST_DIR / "doc_test" / "recursive_dirs" / "src_trace.toml" + + +@pytest.fixture(scope="session") +def source_directory() -> Path: + tests_dir = Path(__file__).parent + source_directory = tests_dir / "data" / "dcdc" + return source_directory + + +@pytest.fixture(scope="session") +def source_paths(source_directory: Path) -> list[Path]: + source_paths = [ + source_directory / "charge" / "demo_1.cpp", + source_directory / "charge" / "demo_2.cpp", + source_directory / "discharge" / "demo_3.cpp", + source_directory / "supercharge.cpp", + ] + return source_paths + + +@pytest.fixture(scope="session", autouse=True) +def temporary_gitignore(source_directory: Path): + gitignore_path = source_directory / ".gitignore" + gitignore_path.write_text("demo_1.cpp\n", encoding="utf-8") + yield + gitignore_path.unlink() + + +class DoctreeSnapshotExtension(SingleFileSnapshotExtension): + _write_mode = WriteMode.TEXT + _file_extension = "doctree.xml" + + def serialize(self, data, **_kwargs): + if not isinstance(data, document): + raise TypeError(f"Expected document, got {type(data)}") + doc = data.deepcopy() + doc["source"] = "" # this will be a temp path + doc.attributes.pop("translation_progress", None) # added in sphinx 7.1 + return doc.pformat() + + +@pytest.fixture +def snapshot_doctree(snapshot): + """Snapshot fixture for doctrees. + + Here we try to sanitize the doctree, to make the snapshots reproducible. + """ + try: + return snapshot.with_defaults(extension_class=DoctreeSnapshotExtension) + except AttributeError: + # fallback for older versions of pytest-snapshot + return snapshot.use_extension(DoctreeSnapshotExtension) diff --git a/tests/data/dcdc/charge/demo_1.cpp b/tests/data/dcdc/charge/demo_1.cpp new file mode 100644 index 0000000..750ffdd --- /dev/null +++ b/tests/data/dcdc/charge/demo_1.cpp @@ -0,0 +1,28 @@ +// demo_1.cpp + +/** + * @file another_example.cpp + * @brief Test file with nested rst blocks. + */ + + #include + + /** + * @brief Function with nested reST blocks. + * + * Include details on how to handle edge cases. + * + * Additional processing steps here. + * [[IMPL_processAssemble, processAssemble function]] + */ + void processAssemble(){ + //... + } + + // [[IMPL_main_demo1, main function]] + int main() { + std::cout << "Starting demo_1..." << std::endl; + processAssemble(); + std::cout << "Demo_1 finished." << std::endl; + return 0; + } diff --git a/tests/data/dcdc/charge/demo_2.cpp b/tests/data/dcdc/charge/demo_2.cpp new file mode 100644 index 0000000..8904a1a --- /dev/null +++ b/tests/data/dcdc/charge/demo_2.cpp @@ -0,0 +1,48 @@ +// demo_2.cpp + +/** + * @file another_example.cpp + * @brief Test file with nested rst blocks. + */ + + #include + + /** + * @brief Function with nested reST blocks. + * + * + * Include details on how to handle edge cases. + * + * Additional processing steps here. + * [[IMPL_filterData, filterData func, impl]] + */ + void filterData() { + // ... implementation ... + } + + /** + * @brief Function with multiple rst blocks. + * + * Some code here. + * + * Feature F - Data visualization + * [[IMPL_processAggregate]] + */ + void processAggregate(){ + //... + } + + /** + * @brief Function with a rst blocks. + * .. impl:: Feature G - Data loss prevention + * + * Some description here. + * [[ IMPL_main_demo2, main func in demo_2]] + */ + int main() { + std::cout << "Starting demo_2..." << std::endl; + filterData(); + processAggregate(); + std::cout << "Demo_2 finished." << std::endl; + return 0; + } diff --git a/tests/data/dcdc/discharge/demo_3.cpp b/tests/data/dcdc/discharge/demo_3.cpp new file mode 100644 index 0000000..9a635fc --- /dev/null +++ b/tests/data/dcdc/discharge/demo_3.cpp @@ -0,0 +1,56 @@ +// demo_3.cpp + +/** + * @file varied_example.cpp + * @brief Test file with varied rst formatting. + */ + + #include + + // GLOBAL REQUIRE: Feature G - Configuration management + // Description: Manage application configuration. + + /** + * @brief Function with varied rst spacing. + * + * + */ + void logErrors() { + // ... implementation ... + } + + /** + * @brief Function with rst on same line. (NOT valid) + * + */ +// [[ IMPL_displayUI, displayUI() func\, so that it displays UI]] + void displayUI(){ + //... + } + + /** + * @brief function with rst with extra space. + * + * \rst + * .. impl:: Feature J - Data backup + * :id: IMPL_6 + * + * Backup user data. + * \endrst + * + * // TODO: Improve backup performance. + * // [[ IMPL_backupData, back up data]] + */ + void backupData(){ + //... + } + + // [[IMPL_main_demo_3, main func in demo_3.cpp]] + int main() { + std::cout << "Starting demo_3..." << std::endl; + logErrors(); + displayUI(); + backupData(); + std::cout << "Demo_3 finished." << std::endl; + return 0; + } diff --git a/tests/data/dcdc/supercharge.cpp b/tests/data/dcdc/supercharge.cpp new file mode 100644 index 0000000..24bc4e2 --- /dev/null +++ b/tests/data/dcdc/supercharge.cpp @@ -0,0 +1,31 @@ + +#include + +// [[IMPL_singleLineExample, singleLineExample func, impl, [IMPL_main_demo_3, IMPL_main_demo2]]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} + +/* + * + * This is a multi-line comment example. + * It spans multiple lines and contains detailed information. + * [[IMPL_multiLineExample, multiLineExample func, impl, [IMPL_main_demo_3, IMPL_main_demo1]]] + */ +void multiLineExample() +{ + std::cout << "Multi-line comment example" << std::endl; +} + +// [[IMPL_14, title 13, impl, 13[\[SPEC\,_1\]], open, low, high]] +// invalid because the too many fields +void baz() {} + +// one-line comment style: [[IMPL_main_supercharge, main func in supercharge.cpp, impl, [IMPL_main_demo_3, IMPL_main_demo1, IMPL_main_demo2]]] +int main() +{ + singleLineExample(); + multiLineExample(); + return 0; +} diff --git a/tests/data/oneline_comment_basic/basic_oneliners.c b/tests/data/oneline_comment_basic/basic_oneliners.c new file mode 100644 index 0000000..9102964 --- /dev/null +++ b/tests/data/oneline_comment_basic/basic_oneliners.c @@ -0,0 +1,29 @@ +// [[IMPL_1, Function Foo]] +void foo() {} + +// [[IMPL_2, Function Bar, impl, [], closed]] +void bar() {} + +// [[IMPL_3, Function Baz\, as I want it, impl, [], closed]] +void baz() {} + +// [[IMPL_5, Function Bar, impl, [SPEC_1, SPEC_2], open]] +void bar() {} + +// [[IMPL_6, Function Bar, impl, [SPEC_1, SPEC_2], [open]]] +// valid because "[open]" is parsed into field status +void foo() {} + +// [[IMPL_7, Function has a, in the title]] +// title has a non-escaped split char ',' +// type will be set to 'in the title' (will lead to a SN error) +void baz() {} + +// [[IMPL_8, [Title starts with a bracket], impl]] +// valid because title is of type string, it will be parsed to +// '[Title starts with a bracket]' +void baz() {} + +// [[IMPL_9, Function Baz, impl, [SPEC_1, SPEC_2[text], SPEC_3], open]] +// valid because the 2nd link item is 'SPEC_2]' +void baz() {} diff --git a/tests/data/oneline_comment_basic/vdoc_config.toml b/tests/data/oneline_comment_basic/vdoc_config.toml new file mode 100644 index 0000000..dec2a20 --- /dev/null +++ b/tests/data/oneline_comment_basic/vdoc_config.toml @@ -0,0 +1,19 @@ +src_dir = "./" +comment_type = "c" +exclude = [] +include = ["**/*.c", "**/*.h"] +gitignore = false + +[oneline_comment_style] +start_sequence = "[[" +end_sequence = "]]" # default is newline character +field_split_char = "," +needs_fields = [ + { "name" = "id" }, + { "name" = "title" }, + { "name" = "type", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, + { "name" = "status", "default" = "open" }, + { "name" = "priority", "default" = "low" }, +] diff --git a/tests/data/oneline_comment_default/default_oneliners.c b/tests/data/oneline_comment_default/default_oneliners.c new file mode 100644 index 0000000..5a7e920 --- /dev/null +++ b/tests/data/oneline_comment_default/default_oneliners.c @@ -0,0 +1,19 @@ +// @Function Foo, IMPL_1 +void foo() {} + +// @Function Bar, IMPL_2 +void bar() {} + +// @Function Baz\, as I want it, IMPL_3 +void baz() {} + +// @Function Bar\, , IMPL_4, impl, [SPEC_1, SPEC_2] +void bar() {} + +/* +* Multiple lines comment +* +* +* @Function Bar, , IMPL_4, impl, [SPEC_1, SPEC_2] +*/ +void bar() {} diff --git a/tests/data/oneline_comment_default/vdoc_config.toml b/tests/data/oneline_comment_default/vdoc_config.toml new file mode 100644 index 0000000..58183fb --- /dev/null +++ b/tests/data/oneline_comment_default/vdoc_config.toml @@ -0,0 +1,5 @@ +src_dir = "./" +comment_type = "c" +exclude = [] +include = ["**/*.c", "**/*.h"] +gitignore = false diff --git a/tests/data/sphinx/Makefile b/tests/data/sphinx/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/tests/data/sphinx/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/tests/data/sphinx/conf.py b/tests/data/sphinx/conf.py new file mode 100644 index 0000000..8842a1c --- /dev/null +++ b/tests/data/sphinx/conf.py @@ -0,0 +1,113 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "test_parse" +copyright = "2025, useblocks" +author = "team useblocks" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "furo" +# html_static_path = ["_static"] + +extensions = ["sphinx_needs", "sphinx_codelinks"] + +needs_types = [ + { + "directive": "story", + "title": "User Story", + "prefix": "US_", + "color": "#BFD8D2", + "style": "node", + }, + { + "directive": "spec", + "title": "Specification", + "prefix": "SP_", + "color": "#FEDCD2", + "style": "node", + }, + { + "directive": "implement", + "title": "Implementation", + "prefix": "IM_", + "color": "#DF744A", + "style": "node", + }, + { + "directive": "impl", + "title": "Impl", + "prefix": "IMPL_", + "color": "#DFd44A", + "style": "node", + }, + { + "directive": "test", + "title": "Test Case", + "prefix": "TC_", + "color": "#DCB239", + "style": "node", + }, +] + +needs_extra_options = ["priority"] + +src_trace_config_from_toml = "src_trace.toml" + +# # TODO implement me +# src_trace_set_local_url = True +# src_trace_local_url_field = "local-url" +# src_trace_set_remote_url = True +# src_trace_remote_url_field = "remote-url" + +# src_trace_projects = { +# # TODO use the key (add it to the src-trace need) +# "dcdc": { +# "type": "cpp", +# "src_dir": "../../dcdc", # relative to confdir +# "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}", # optional +# "exclude": ["dcdc/src/ubt/ubt.cpp"], +# "include": ["**/*.cpp", "**/*.hpp"], # has default for each type +# "gitignore": True, # default is True +# # Proposal for the one-line comment style: +# # a need object defined in a one-line comment with the customized style. +# # The example is the following: +# # [[directive: implement, title: charge, id:impl_charge, link: req_charge]]]] +# # The equivalent need object in rst is: +# # .. implement:: implement charge +# # :id: impl_charge +# # :link: req_charge +# "oneline_comment_style": { +# "start": "[[", +# "end": "]]", +# "option_separator": ",", +# "key_value_separator": ":", +# ## What's the point if the comment has no readability? +# # "default-need-type": "implements", +# # "structure": [ +# # "id", +# # "link-type", +# # "link-id", +# # "title", +# # ], +# }, +# "multiline_comment_style": { +# "line-start-char": "*", +# "start": "[[[", +# "end": "]]]", +# }, +# } +# } diff --git a/tests/data/sphinx/index.rst b/tests/data/sphinx/index.rst new file mode 100644 index 0000000..e0d730b --- /dev/null +++ b/tests/data/sphinx/index.rst @@ -0,0 +1,11 @@ +.. src-trace:: dcdc_supercharge + :project: dcdc + :file: supercharge.cpp + +.. src-trace:: dcdc_charge + :project: dcdc + :directory: ./charge + +.. src-trace:: dcdc_discharge + :project: dcdc + :directory: ./discharge diff --git a/tests/data/sphinx/make.bat b/tests/data/sphinx/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/tests/data/sphinx/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/tests/data/sphinx/src_trace.toml b/tests/data/sphinx/src_trace.toml new file mode 100644 index 0000000..cba8aa2 --- /dev/null +++ b/tests/data/sphinx/src_trace.toml @@ -0,0 +1,28 @@ +[src_trace] +set_local_url = true +local_url_field = "local-url" +set_remote_url = true +remote_url_field = "remote-url" +debug_measurement = true + +[src_trace.projects.dcdc] +comment_type = "cpp" +src_dir = "../dcdc" +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" +exclude = ["dcdc/src/ubt/ubt.cpp"] +include = ["**/*.cpp", "**/*.hpp"] +gitignore = true + +[src_trace.projects.dcdc.oneline_comment_style] +start_sequence = "[[" +end_sequence = "]]" # default is newline character +field_split_char = "," +needs_fields = [ + { "name" = "id" }, + { "name" = "title" }, + { "name" = "type", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, + { "name" = "status", "default" = "open" }, + { "name" = "priority", "default" = "low" }, +] diff --git a/tests/doc_test/recursive_dirs/conf.py b/tests/doc_test/recursive_dirs/conf.py new file mode 100644 index 0000000..2c6a754 --- /dev/null +++ b/tests/doc_test/recursive_dirs/conf.py @@ -0,0 +1,66 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "test_parse" +copyright = "2025, useblocks" +author = "team useblocks" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +# html_static_path = ["_static"] + +extensions = ["sphinx_needs", "sphinx_codelinks"] + +needs_types = [ + { + "directive": "story", + "title": "User Story", + "prefix": "US_", + "color": "#BFD8D2", + "style": "node", + }, + { + "directive": "spec", + "title": "Specification", + "prefix": "SP_", + "color": "#FEDCD2", + "style": "node", + }, + { + "directive": "implement", + "title": "Implementation", + "prefix": "IM_", + "color": "#DF744A", + "style": "node", + }, + { + "directive": "impl", + "title": "Impl", + "prefix": "IMPL_", + "color": "#DFd44A", + "style": "node", + }, + { + "directive": "test", + "title": "Test Case", + "prefix": "TC_", + "color": "#DCB239", + "style": "node", + }, +] + +src_trace_config_from_toml = "src_trace.toml" diff --git a/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_1.cpp b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_1.cpp new file mode 100644 index 0000000..fa31b64 --- /dev/null +++ b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_1.cpp @@ -0,0 +1,8 @@ + +#include + +// [[ directive:implement, title: implement req 1, id: IMPL_1, link: REQ_1 ]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} diff --git a/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_2.cpp b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_2.cpp new file mode 100644 index 0000000..5c2d046 --- /dev/null +++ b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_2.cpp @@ -0,0 +1,8 @@ + +#include + +// [[ directive:implement, title: implement req 2, id: IMPL_2, link: REQ_2 ]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} diff --git a/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_3.cpp b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_3.cpp new file mode 100644 index 0000000..9bd7c34 --- /dev/null +++ b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_3.cpp @@ -0,0 +1,8 @@ + +#include + +// [[ directive:implement, title: implement req 3, id: IMPL_3, link: REQ_3 ]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} diff --git a/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_lv4/dummy_4.cpp b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_lv4/dummy_4.cpp new file mode 100644 index 0000000..c60debf --- /dev/null +++ b/tests/doc_test/recursive_dirs/dummy_src_lv1/dummy_lv2/dummy_lv3/dummy_lv4/dummy_4.cpp @@ -0,0 +1,8 @@ + +#include + +// [[ directive:implement, title: implement req 4, id: IMPL_4, link: REQ_4 ]] +void singleLineExample() +{ + std::cout << "Single-line comment example" << std::endl; +} diff --git a/tests/doc_test/recursive_dirs/index.rst b/tests/doc_test/recursive_dirs/index.rst new file mode 100644 index 0000000..9a1dc96 --- /dev/null +++ b/tests/doc_test/recursive_dirs/index.rst @@ -0,0 +1,2 @@ +.. src-trace:: dummy src + :project: dummy_src diff --git a/tests/doc_test/recursive_dirs/src_trace.toml b/tests/doc_test/recursive_dirs/src_trace.toml new file mode 100644 index 0000000..077df29 --- /dev/null +++ b/tests/doc_test/recursive_dirs/src_trace.toml @@ -0,0 +1,26 @@ +[src_trace] +set_local_url = true +local_url_field = "local-url" +set_remote_url = true +remote_url_field = "remote-url" +debug_measurement = true + +[src_trace.projects.dummy_src] +comment_type = "cpp" +src_dir = "./dummy_src_lv1" +remote_url_pattern = "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}" +exclude = ["dcdc/src/ubt/ubt.cpp"] +include = ["**/*.cpp", "**/*.hpp"] +gitignore = true + +[src_trace.projects.dummy_src.oneline_comment_style] +start_sequence = "[[" +end_sequence = "]]" # default is newline character +field_split_char = "," +needs_fields = [ + { "name" = "id", "type" = "str" }, + { "name" = "title", "type" = "str" }, + { "name" = "type", "type" = "str", "default" = "impl" }, + { "name" = "links", "type" = "list[str]", "default" = [ + ] }, +] diff --git a/tests/test_cmd.py b/tests/test_cmd.py new file mode 100644 index 0000000..fa31858 --- /dev/null +++ b/tests/test_cmd.py @@ -0,0 +1,249 @@ +from pathlib import Path + +import pytest +import toml +from typer.testing import CliRunner + +from sphinx_codelinks.cmd import app + +from .conftest import ( + BASIC_VDOC_TOML, + DEFAULT_VDOC_TOML, + RECURSIVE_DIR_VDOC_TOML, + SRC_TRACE_TOML, + TEST_DIR, +) + +ONELINE_COMMENT_TEMPLATE = { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + {"name": "id"}, + {"name": "title"}, + {"name": "type"}, + ], +} + +VDOC_CONFIG_TEMPLATE = { + "src_dir": str(TEST_DIR / "data" / "dcdc"), + "exclude": ["**/charge/demo_1.cpp", "**/discharge/demo_3.cpp"], + "include": ["**/charge/demo_2.cpp", "**/supercharge.cpp"], + "gitignore": True, + "file_types": ["cpp"], + "oneline_comment_style": ONELINE_COMMENT_TEMPLATE, +} + + +runner = CliRunner() + + +@pytest.mark.parametrize( + ("options", "stdout"), + [ + ( + ["discover", str(TEST_DIR / "data" / "dcdc"), "--no-gitignore"], + "5 files discovered", + ), + ( + ["discover", str(TEST_DIR / "data" / "dcdc"), "--gitignore"], + "4 files discovered", + ), + ], +) +def test_discover(options, stdout): + result = runner.invoke(app, options) + assert result.exit_code == 0 + assert stdout in result.stdout + + +@pytest.mark.parametrize( + ("options", "lines"), + [ + ( + [ + "vdoc", + "--config", + SRC_TRACE_TOML, + "--project", + "dcdc", + ], + [ + "The virtual documents are generated:", + Path("charge") / "demo_1.json", + Path("charge") / "demo_2.json", + Path("discharge") / "demo_3.json", + Path("supercharge.json"), + "The cached files are:", + TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", + TEST_DIR / "data" / "dcdc" / "charge" / "demo_2.cpp", + TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", + TEST_DIR / "data" / "dcdc" / "supercharge.cpp", + ], + ), + ( + [ + "vdoc", + "--config", + BASIC_VDOC_TOML, + ], + [ + "The virtual documents are generated:", + Path("basic_oneliners.json"), + "The cached files are:", + TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", + ], + ), + ( + [ + "vdoc", + "--config", + DEFAULT_VDOC_TOML, + ], + [ + "The virtual documents are generated:", + Path("default_oneliners.json"), + "The cached files are:", + TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", + ], + ), + ( + ["vdoc", "--config", RECURSIVE_DIR_VDOC_TOML, "--project", "dummy_src"], + [ + "The virtual documents are generated:", + Path("dummy_1.json"), + Path("dummy_lv2") / "dummy_2.json", + Path("dummy_lv2") / "dummy_lv3" / "dummy_3.json", + Path("dummy_lv2") / "dummy_lv3" / "dummy_lv4" / "dummy_4.json", + "The cached files are:", + TEST_DIR + / "doc_test" + / "recursive_dirs" + / "dummy_src_lv1" + / "dummy_1.cpp", + TEST_DIR + / "doc_test" + / "recursive_dirs" + / "dummy_src_lv1" + / "dummy_lv2" + / "dummy_2.cpp", + TEST_DIR + / "doc_test" + / "recursive_dirs" + / "dummy_src_lv1" + / "dummy_lv2" + / "dummy_lv3" + / "dummy_3.cpp", + TEST_DIR + / "doc_test" + / "recursive_dirs" + / "dummy_src_lv1" + / "dummy_lv2" + / "dummy_lv3" + / "dummy_lv4" + / "dummy_4.cpp", + ], + ), + ], +) +def test_vdoc(options, lines, tmp_path): + options.append("--output-dir") + options.append(tmp_path) + for i in range(len(lines)): + if lines[i] == "The virtual documents are generated:": + continue + if lines[i] == "The cached files are:": + break + lines[i] = tmp_path / lines[i] + + lines = [str(line) for line in lines] + + result = runner.invoke( + app, + options, + ) + + assert result.exit_code == 0 + assert result.stdout.splitlines() == lines + + +@pytest.mark.parametrize( + ("config_dict", "output_lines"), + [ + ( + { + key: (123 if key == "exclude" else value) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Invalid value: Invalid source discovery configuration:", + "Schema validation error in field 'exclude': 123 is not of type 'array'", + ], + ), + ( + { + key: (123 if key == "include" else value) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Invalid value: Invalid source discovery configuration:", + "Schema validation error in field 'include': 123 is not of type 'array'", + ], + ), + ( + { + key: (123 if key in ("exclude", "include", "src_dir") else value) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Invalid value: Invalid source discovery configuration:", + "src_dir must be a string", + "Schema validation error in field 'exclude': 123 is not of type 'array'", + "Schema validation error in field 'include': 123 is not of type 'array'", + ], + ), + ( + { + key: ( + {"not_expected": 123} if key == "oneline_comment_style" else value + ) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Invalid value: Invalid oneline comment style configuration:", + "OneLineCommentStyle.__init__() got an unexpected keyword argument", + "'not_expected'", + ], + ), + ( + { + key: ( + {"needs_fields": [{"name": "id"}, {"name": "id"}]} + if key == "oneline_comment_style" + else value + ) + for key, value in VDOC_CONFIG_TEMPLATE.items() + }, + [ + "Invalid value: Invalid oneline comment style configuration:", + "Missing required fields: ['title', 'type']", + "Field 'id' is defined multiple times.", + ], + ), + ], +) +def test_vdoc_config_negative(config_dict, output_lines, tmp_path: Path) -> None: + # Force disable Rich styling + config_file = tmp_path / "vdoc_config.toml" + with config_file.open("w", encoding="utf-8") as f: + toml.dump(config_dict, f) + + options = [ + "vdoc", + "--config", + str(config_file), + ] + result = runner.invoke(app, options) + assert result.exit_code != 0 + for line in output_lines: + assert line in result.stderr diff --git a/tests/test_source_discover.py b/tests/test_source_discover.py new file mode 100644 index 0000000..522d0b6 --- /dev/null +++ b/tests/test_source_discover.py @@ -0,0 +1,95 @@ +from pathlib import Path + +import pytest + +from sphinx_codelinks.source_discovery.config import SourceDiscoveryConfig +from sphinx_codelinks.source_discovery.source_discover import SourceDiscover + + +@pytest.mark.parametrize( + ("config", "msgs"), + [ + ( + { + "root_dir": 123, + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "file_types": ["cpp", "hpp"], + }, + [ + "Schema validation error in field 'root_dir': 123 is not of type 'string'" + ], + ), + ( + { + "root_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": "TrueAsString", + "file_types": ["cpp", "hpp"], + }, + [ + "Schema validation error in field 'gitignore': 'TrueAsString' is not of type 'boolean'" + ], + ), + ], +) +def test_schema_negative(config, msgs): + source_discovery_config = SourceDiscoveryConfig(**config) + errors = source_discovery_config.check_schema() + assert errors.sort() == msgs.sort() + + +@pytest.mark.parametrize( + "config", + [ + {}, + { + "root_dir": "/path/to/root", + "exclude": ["exclude1", "exclude2"], + "include": ["include1", "include2"], + "gitignore": True, + "file_types": ["cpp", "hpp"], + }, + ], +) +def test_schema_positive(config): + source_discovery_config = SourceDiscoveryConfig(**config) + errors = source_discovery_config.check_schema() + assert len(errors) == 0 + + +def test_source_discover_all_files(source_directory: Path): + source_discover = SourceDiscover(source_directory, gitignore=False) + assert len(source_discover.source_paths) == 5 + + +def test_source_discover_gitignore(source_directory: Path): + source_discover = SourceDiscover(source_directory, gitignore=True) + assert len(source_discover.source_paths) == 4 + + +def test_source_discover_includes(source_directory: Path): + source_discover = SourceDiscover( + source_directory, + gitignore=True, + exclude=["charge/*.cpp"], + include=["**/*.cpp"], + ) + assert len(source_discover.source_paths) == 5 + + +def test_source_discover_excludes(source_directory: Path): + source_discover = SourceDiscover( + source_directory, gitignore=True, exclude=["charge/*.cpp"] + ) + assert len(source_discover.source_paths) == 3 + + +def test_source_discover_type(source_directory: Path): + source_discover = SourceDiscover( + source_directory, gitignore=False, file_types=["cpp"] + ) + assert len(source_discover.source_paths) == 4 + assert all(path.suffix == ".cpp" for path in source_discover.source_paths) diff --git a/tests/test_src_trace.py b/tests/test_src_trace.py new file mode 100644 index 0000000..41405a6 --- /dev/null +++ b/tests/test_src_trace.py @@ -0,0 +1,203 @@ +from collections.abc import Callable +from pathlib import Path +import shutil + +import pytest +from sphinx.testing.util import SphinxTestApp + +from sphinx_codelinks.sphinx_extension.config import ( + SrcTraceSphinxConfig, + check_configuration, +) +from sphinx_codelinks.sphinx_extension.source_tracing import set_config_to_sphinx + + +@pytest.mark.parametrize( + ("src_trace_config", "result"), + [ + ( + { + "remote_url_field": 555, + "local_url_field": 789, + "set_local_url": "fdd", + "set_remote_url": "TrueString", + "projects": { + "dcdc": { + "comment_type": "java", + "src_dir": ["../dcdc"], + "remote_url_pattern": 44332, + "exclude": [123], + "include": [345], + "gitignore": "_true", + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "list[]", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, + } + }, + }, + [ + "Schema validation error in filed 'remote_url_field': 555 is not of type 'string'", + "Schema validation error in filed 'set_remote_url': 'TrueString' is not of type 'boolean'", + "Schema validation error in filed 'local_url_field': 789 is not of type 'string'", + "Schema validation error in filed 'set_local_url': 'fdd' is not of type 'boolean'", + "Project 'dcdc' has the following errors:", + "Schema validation error in need_fields 'title': 'list[]' is not one of ['str', 'list[str]']", + "src_dir must be a string", + "comment_type must be one of ['c', 'cpp', 'h', 'hpp']", + "Schema validation error in field 'exclude': 123 is not of type 'string'", + "Schema validation error in field 'include': 345 is not of type 'string'", + "Schema validation error in field 'gitignore': '_true' is not of type 'boolean'", + "remote_url_pattern must be a string", + ], + ), + ( + { + "remote_url_field": "remote-url", + "local_url_field": "local-url", + "set_local_url": True, + "set_remote_url": True, + "projects": { + "dcdc": { + "comment_type": "cpp", + "src_dir": "../dcdc", + # intentionally not given "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}", + "exclude": [], + "include": [], + "gitignore": True, + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "str", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, + } + }, + }, + [ + "Project 'dcdc' has the following errors:", + "remote_url_pattern must be given, as set_remote_url is enabled", + ], + ), + ], +) +def test_src_tracing_config_negative( + make_app: Callable[..., SphinxTestApp], + src_trace_config, + result, +): + this_file_dir = Path(__file__).parent + sphinx_project = Path("data") / "sphinx" + app = make_app(srcdir=(this_file_dir / sphinx_project)) + set_config_to_sphinx(src_trace_config, app.env.config) + src_trace_sphinx_config = SrcTraceSphinxConfig(app.env.config) + errors = check_configuration(src_trace_sphinx_config) + assert sorted(errors) == sorted(result) + + +def test_src_tracing_config_positive( + make_app: Callable[..., SphinxTestApp], +): + src_trace_config = { + "remote_url_field": "remote-url", + "local_url_field": "local-url", + "set_local_url": True, + "set_remote_url": True, + "projects": { + "dcdc": { + "comment_type": "cpp", + "src_dir": "../dcdc", + "remote_url_pattern": "https://github.com/useblocks/sphinx-codelinks/blob/{commit}/{path}#L{line}", + "exclude": ["**/*.hpp"], + "include": ["**/*.cpp"], + "gitignore": True, + "oneline_comment_style": { + "start_sequence": "[[", + "end_sequence": "]]", + "field_split_char": ",", + "needs_fields": [ + { + "name": "title", + "type": "str", + }, + { + "name": "type", + "default": "impl", + "type": "str", + }, + ], + }, + } + }, + } + this_file_dir = Path(__file__).parent + sphinx_project = Path("data") / "sphinx" + app = make_app(srcdir=(this_file_dir / sphinx_project)) + set_config_to_sphinx(src_trace_config, app.env.config) + src_trace_sphinx_config = SrcTraceSphinxConfig(app.env.config) + errors = check_configuration(src_trace_sphinx_config) + assert not errors + + +@pytest.mark.parametrize( + ("sphinx_project", "source_code"), + [ + (Path("data") / "sphinx", Path("data") / "dcdc"), + ( + Path("doc_test") / "recursive_dirs", + Path("doc_test") / "recursive_dirs" / "dummy_src_lv1", + ), + ], +) +def test_build_html( + tmpdir: Path, + make_app: Callable[..., SphinxTestApp], + sphinx_project, + source_code, + snapshot_doctree, +): + this_file_dir = Path(__file__).parent + + sphinx_src_dir = tmpdir / sphinx_project + shutil.copytree( + this_file_dir / sphinx_project, + sphinx_src_dir, + dirs_exist_ok=True, + ) + shutil.copytree( + this_file_dir / source_code, + tmpdir / source_code, + dirs_exist_ok=True, + ) + + app: SphinxTestApp = make_app( + srcdir=Path(sphinx_src_dir), + freshenv=True, + ) + app.build() + html = Path(app.outdir, "index.html").read_text() + + assert html + assert app.env.get_doctree("index") == snapshot_doctree diff --git a/tests/test_virtual_docs.py b/tests/test_virtual_docs.py new file mode 100644 index 0000000..6bb7d32 --- /dev/null +++ b/tests/test_virtual_docs.py @@ -0,0 +1,548 @@ +import os + +import pytest + +from sphinx_codelinks.virtual_docs.config import ( + ESCAPE, + OneLineCommentStyle, + VirtualDocsConfig, +) +from sphinx_codelinks.virtual_docs.utils import ( + OnelineParserInvalidWarning, + WarningSubTypeEnum, + oneline_parser, +) +from sphinx_codelinks.virtual_docs.virtual_docs import VirtualDocs + +from .conftest import TEST_DIR + +ONELINE_COMMENT_STYLE = OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "id"}, + {"name": "title"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + {"name": "status", "default": "open"}, + {"name": "priority", "default": "low"}, + ], +) + +ONELINE_COMMENT_STYLE_DEFAULT = OneLineCommentStyle() + + +@pytest.mark.parametrize( + ("vdocs_config", "result"), + [ + ( + VirtualDocsConfig( + src_files=[ + TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", + ], + src_dir=TEST_DIR / "data" / "dcdc", + output_dir=TEST_DIR / "output", + comment_type=123, + ), + [ + "Schema validation error in field 'comment_type': 123 is not of type 'string'", + ], + ), + ( + VirtualDocsConfig( + src_files=None, + src_dir=TEST_DIR / "data" / "dcdc", + output_dir=TEST_DIR / "output", + comment_type=123, + ), + [ + "Schema validation error in field 'comment_type': 123 is not of type 'string'", + "Schema validation error in field 'src_files': None is not of type 'array'", + ], + ), + ], +) +def test_config_schema_validator_negative(vdocs_config, result): + errors = vdocs_config.check_schema() + assert sorted(errors) == sorted(result) + + +@pytest.mark.parametrize( + "oneline_config, result", + [ + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[]", "default": []}, # wrong type + ], + ), + [ + "Schema validation error in need_fields 'links': 'list[]' is not one of ['str', 'list[str]']" + ], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": 123}, # int is invalid + {"name": "links", "type": "list[str]", "default": []}, + ], + ), + [ + "Schema validation error in need_fields 'type': 123 is not of type 'string'" + ], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title", "qwe": "qwe"}, # invalid qwe filed + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + ), + [ + "Schema validation error in need_fields 'title': Additional properties are not allowed ('qwe' was unexpected)" + ], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + { + "name": "type", + "type: ": "list[str]", + "default": "impl", + }, # wring combination of type and default + {"name": "links", "type": "list[str]", "default": []}, + ], + ), + [ + "Schema validation error in need_fields 'type': Additional properties are not allowed ('type: ' was unexpected)" + ], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "id"} # "title" and "type" are not given + ], + ), + ["Missing required fields: ['title', 'type']"], + ), + ( + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "id"}, + {"name": "id"}, # duplicate + ], + ), + [ + "Missing required fields: ['title', 'type']", + "Field 'id' is defined multiple times.", + ], + ), + ( + OneLineCommentStyle( + start_sequence=1234, # wrong type + end_sequence=5678, + field_split_char=2222, + needs_fields=[ + {"name": "id"}, + ], + ), + [ + "Schema validation error in field 'field_split_char': 2222 is not of type 'string'", + "Schema validation error in field 'end_sequence': 5678 is not of type 'string'", + "Schema validation error in field 'start_sequence': 1234 is not of type 'string'", + "Missing required fields: ['title', 'type']", + ], + ), + ], +) +def test_oneline_schema_validator_negative(oneline_config, result): + errors = oneline_config.check_fields_configuration() + assert sorted(errors) == sorted(result) + + +@pytest.mark.parametrize( + "oneline_config", + [ + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, + {"name": "id"}, + {"name": "type", "default": "impl"}, + {"name": "links", "type": "list[str]", "default": []}, + ], + ), + OneLineCommentStyle( + start_sequence="[[", + end_sequence="]]", + field_split_char=",", + needs_fields=[ + {"name": "title"}, # minimum need_fields config + {"name": "type"}, + ], + ), + OneLineCommentStyle( + needs_fields=[ # minimum config + {"name": "title"}, + {"name": "type"}, + ], + ), + ], +) +def test_oneline_schema_validator_positive(oneline_config): + assert len(oneline_config.check_fields_configuration()) == 0 + + +@pytest.mark.parametrize( + "oneline, result", + [ + ( + "[[IMPL_1, title 1]]", + { + "id": "IMPL_1", + "title": "title 1", + "type": "impl", + "links": [], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_2, title 2, impl, [], closed]]", + { + "id": "IMPL_2", + "title": "title 2", + "type": "impl", + "links": [], + "status": "closed", + "priority": "low", + }, + ), + ( + "[[IMPL_3, title\, 3, impl, [], closed]]", + { + "id": "IMPL_3", + "title": "title, 3", + "type": "impl", + "links": [], + "status": "closed", + "priority": "low", + }, + ), + ( + "[[IMPL_5, title 5, impl, [SPEC_1, SPEC_2], open]]", + { + "id": "IMPL_5", + "title": "title 5", + "type": "impl", + "links": ["SPEC_1", "SPEC_2"], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_7, Function has a, in the title]]", + { + "id": "IMPL_7", + "title": "Function has a", + "type": "in the title", + "links": [], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_8, [Title starts with a bracket], impl]]", + { + "id": "IMPL_8", + "title": "[Title starts with a bracket]", + "type": "impl", + "links": [], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_9, Function Baz, impl, [SPEC_1, SPEC_2[text], SPEC_3], open]]", + { + "id": "IMPL_9", + "title": "Function Baz", + "type": "impl", + "links": ["SPEC_1", "SPEC_2[text"], + "status": "SPEC_3]", + "priority": "open", + }, + ), + ( + "[[IMPL_10, title 10, impl, [SPEC_1], open]]", + { + "id": "IMPL_10", + "title": "title 10", + "type": "impl", + "links": ["SPEC_1"], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_11, title 11, impl, [SPEC\,_1], open]]", + { + "id": "IMPL_11", + "title": "title 11", + "type": "impl", + "links": ["SPEC,_1"], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_12, title 12, impl, [\[SPEC\,_1\]], open]]", + { + "id": "IMPL_12", + "title": "title 12", + "type": "impl", + "links": ["[SPEC,_1]"], + "status": "open", + "priority": "low", + }, + ), + ( + "[[IMPL_13, title\\ 13, impl, [\[SPEC\,_1\]], open]]", + { + "id": "IMPL_13", + "title": "title\ 13", + "type": "impl", + "links": ["[SPEC,_1]"], + "status": "open", + "priority": "low", + }, + ), + ], +) +def test_oneline_parser_custom_config_positive(oneline: str, result): + assert oneline_parser(oneline, ONELINE_COMMENT_STYLE) == result + + +@pytest.mark.parametrize( + "oneline, result", + [ + ( + f"@title 1, IMPL_1 {os.linesep}", + { + "title": "title 1", + "id": "IMPL_1", + "type": "impl", + "links": [], + }, + ), + ], +) +def test_oneline_parser_default_config_positive(oneline: str, result): + assert oneline_parser(oneline, ONELINE_COMMENT_STYLE_DEFAULT) == result + + +@pytest.mark.parametrize( + "oneline, result", + [ + ( + f"[[IMPL_4, title{ESCAPE}{ESCAPE}, 4, impl, [], closed]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.missing_square_brackets, + msg="Field links with 'type': 'list[str]' must be given with '[]' brackets", + ), + ), + ( + "[[IMPL_2, Function Bar, impl, [SPEC_1, SPEC_2, open]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.missing_square_brackets, + msg="Field links with 'type': 'list[str]' must be given with '[]' brackets", + ), + ), + ( + "[[IMPL_13, title 13, impl, 13[\[SPEC\,_1\]], open]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.not_start_or_end_with_square_brackets, + msg="Field links with 'type': 'list[str]' must start with '[' and end with ']'", + ), + ), + ( + "[[IMPL_14, title 13, impl, 13[\[SPEC\,_1\]], open, low, high]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.too_many_fields, + msg="7 given fields. They shall be less than 6", + ), + ), + ( + "[[IMPL_15]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.too_few_fields, + msg="1 given fields. They shall be more than 2", + ), + ), + ( + f"[[IMPL_16]]{os.linesep}, title 16]]", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.newline_in_field, + msg="Field id has newline character. It is not allowed", + ), + ), + ], +) +def test_oneline_parser_custom_config_negative( + oneline: str, result: OnelineParserInvalidWarning +): + res = oneline_parser(oneline, ONELINE_COMMENT_STYLE) + assert res == result + + +@pytest.mark.parametrize( + "oneline, result", + [ + ( + f"@title 17]]{os.linesep}, IMPL_17 {os.linesep}", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.newline_in_field, + msg="Field title has newline character. It is not allowed", + ), + ), + ( + f"@title 17]], IMPL_17, impl, [SPEC_3, SPEC_4{os.linesep} ] {os.linesep}", + OnelineParserInvalidWarning( + sub_type=WarningSubTypeEnum.newline_in_field, + msg="Field links has newline character. It is not allowed", + ), + ), + ], +) +def test_oneline_parser_default_config_negative(oneline: str, result): + assert oneline_parser(oneline, ONELINE_COMMENT_STYLE_DEFAULT) == result + + +@pytest.mark.parametrize( + "src_dir, src_paths , oneline_comment_style, result", + [ + ( + TEST_DIR / "data" / "dcdc", + [ + TEST_DIR / "data" / "dcdc" / "charge" / "demo_1.cpp", + TEST_DIR / "data" / "dcdc" / "charge" / "demo_2.cpp", + TEST_DIR / "data" / "dcdc" / "discharge" / "demo_3.cpp", + TEST_DIR / "data" / "dcdc" / "supercharge.cpp", + ], + ONELINE_COMMENT_STYLE, + { + "num_virtual_docs": 4, + "num_src_files": 4, + "num_uncached_files": 4, + "num_cached_files": 0, + "num_valid_comments": 10, + "num_oneline_warnings": 2, + }, + ), + ( + TEST_DIR / "data" / "oneline_comment_basic", + [ + TEST_DIR / "data" / "oneline_comment_basic" / "basic_oneliners.c", + ], + ONELINE_COMMENT_STYLE, + { + "num_virtual_docs": 1, + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_valid_comments": 8, + "num_oneline_warnings": 0, + "warnings_path_exists": True, + }, + ), + ( + TEST_DIR / "data" / "oneline_comment_default", + [ + TEST_DIR / "data" / "oneline_comment_default" / "default_oneliners.c", + ], + ONELINE_COMMENT_STYLE_DEFAULT, + { + "num_virtual_docs": 1, + "num_src_files": 1, + "num_uncached_files": 1, + "num_cached_files": 0, + "num_valid_comments": 4, + "num_oneline_warnings": 1, + "warnings_path_exists": True, + }, + ), + ], +) +def test_virtual_docs(tmp_path, src_dir, src_paths, oneline_comment_style, result): + virtual_docs = VirtualDocs(src_paths, src_dir, tmp_path, oneline_comment_style) + virtual_docs.collect() + + assert len(virtual_docs.virtual_docs) == result["num_virtual_docs"] + assert len(virtual_docs.src_files) == result["num_src_files"] + assert len(virtual_docs.cache.uncached_files) == result["num_uncached_files"] + assert len(virtual_docs.cache.cached_files) == result["num_cached_files"] + assert len(virtual_docs.oneline_warnings) == result["num_oneline_warnings"] + assert virtual_docs.warnings_path.exists() + + loaded_warnings = VirtualDocs.load_warnings(tmp_path) + + cnt_comments = 0 + for virtual_doc in virtual_docs.virtual_docs: + cnt_comments += len(virtual_doc.comments) + assert cnt_comments == result["num_valid_comments"] + + # generate virtual documents + virtual_docs.dump_virtual_docs() + for src_file in src_paths: + assert (tmp_path / src_file.with_suffix(".json").relative_to(src_dir)).exists() + + # cache + virtual_docs.cache.update_cache() + assert len(virtual_docs.cache.cached_files) == result["num_uncached_files"] + assert len(virtual_docs.cache.uncached_files) == result["num_cached_files"] + cache_file = tmp_path / "ubt_cache.json" + assert cache_file.exists() + + # save the current virtual documents + saved_virtual_docs = virtual_docs.virtual_docs + + # use cache + del virtual_docs + virtual_docs = VirtualDocs(src_paths, src_dir, tmp_path, oneline_comment_style) + virtual_docs.collect() + assert len(virtual_docs.cache.cached_files) == result["num_uncached_files"] + assert len(virtual_docs.cache.uncached_files) == result["num_cached_files"] + cache_file = tmp_path / "ubt_cache.json" + assert cache_file.exists() + assert VirtualDocs.load_warnings(tmp_path) == loaded_warnings + assert virtual_docs.virtual_docs == saved_virtual_docs