diff --git a/codewiki/src/be/agent_orchestrator.py b/codewiki/src/be/agent_orchestrator.py index d538e678..5a2113b3 100644 --- a/codewiki/src/be/agent_orchestrator.py +++ b/codewiki/src/be/agent_orchestrator.py @@ -1,3 +1,13 @@ +"""Agent orchestration for documentation generation. + +This module defines AgentOrchestrator, the component responsible for +creating and running pydantic_ai agents that generate documentation for +modules discovered in a repository. It selects agent configurations based +on module complexity, wires up the required tools and dependencies, and +drives the per-module documentation generation pipeline (loading/saving the +module tree, invoking the agent, and persisting generated docs). +""" + from pydantic_ai import Agent from pydantic_ai.usage import UsageLimits # import logfire @@ -61,9 +71,6 @@ class AgentOrchestrator: """Orchestrates the AI agents for documentation generation.""" def __init__(self, config: Config): - import logging - logger = logging.getLogger(__name__) - self.config = config self.fallback_models = create_fallback_models(config) self.custom_instructions = config.get_prompt_addition() if config else None @@ -207,4 +214,4 @@ async def process_module(self, module_name: str, components: Dict[str, Node], except Exception as e: logger.error(f"❌ Error processing module {module_name}: {str(e)}") logger.error(f" └─ Traceback: {traceback.format_exc()}") - raise \ No newline at end of file + raise diff --git a/codewiki/src/be/agent_tools/deps.py b/codewiki/src/be/agent_tools/deps.py index fd0bb015..6e51b9af 100644 --- a/codewiki/src/be/agent_tools/deps.py +++ b/codewiki/src/be/agent_tools/deps.py @@ -6,6 +6,7 @@ pipeline. """ from dataclasses import dataclass +from typing import Any from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.config import Config @@ -17,7 +18,7 @@ class CodeWikiDeps: components: dict[str, Node] path_to_current_module: list[str] current_module_name: str - module_tree: dict[str, any] + module_tree: dict[str, Any] max_depth: int current_depth: int config: Config # LLM configuration diff --git a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py index ff717269..aa4272e8 100644 --- a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py +++ b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py @@ -1,3 +1,21 @@ +"""Sub-module documentation generation pipeline step. + +This module implements the recursive agent-dispatch tool used by CodeWiki to +split a module into smaller sub-modules and generate documentation for each +one. It is responsible for: + +- Normalizing the component identifiers returned by the LLM (which may be + either FQDN strings or integer IDs from the ID-based clustering system) + into canonical FQDNs that exist in ``deps.components``. +- Updating the in-memory module tree with the newly created sub-modules. +- Spawning nested ``pydantic_ai`` agents (leaf or non-leaf, depending on + module complexity and depth) to recursively generate documentation for + each sub-module. + +It is exposed to the top-level documentation agent as +``generate_sub_module_documentation_tool``. +""" + from pydantic_ai import RunContext, Tool, Agent from pydantic_ai.usage import UsageLimits @@ -147,28 +165,22 @@ async def generate_sub_module_documentation( description="""Generate detailed documentation for sub-modules by grouping related components. CRITICAL FORMAT REQUIREMENTS: -- Use the EXACT component identifiers as shown in the section +- Use the EXACT integer component IDs as shown in the section - DO NOT extract just class names (e.g., "AuthService", "ApiApplicationConfig") -- Use the COMPLETE identifiers like: "main-repo.src/services/auth.py::AuthService" +- DO NOT invent full FQDN strings; use the integer IDs assigned to each component Example CORRECT format: { - "Authentication": [ - "main-repo.src/services/auth.py::AuthService", - "main-repo.src/services/auth.py::LoginController" - ], - "Configuration": [ - "main-repo.src/config/api.py::ApiApplicationConfig", - "main-repo.src/config/security.py::SecurityConfig" - ] + "Authentication": [0, 1], + "Configuration": [2, 3] } Example WRONG format (DO NOT USE): { "Authentication": ["AuthService", "LoginController"], # ❌ Class names only - "Configuration": ["ApiApplicationConfig"] # ❌ Missing full path + "Configuration": ["main-repo.src/config/api.py::ApiApplicationConfig"] # ❌ Full FQDN string instead of integer ID } -The component identifiers must match exactly what appears in .""", +The integer IDs must match exactly what appears in .""", takes_ctx=True ) diff --git a/codewiki/src/be/dependency_analyzer/analysis/cloning.py b/codewiki/src/be/dependency_analyzer/analysis/cloning.py index 99e37d15..df36942f 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/cloning.py +++ b/codewiki/src/be/dependency_analyzer/analysis/cloning.py @@ -1,3 +1,13 @@ +"""Repository cloning and cleanup utilities. + +This module implements the repository acquisition step of the dependency +analysis pipeline: given a GitHub URL, it sanitizes and validates the URL, +clones the repository into a temporary directory for analysis, and safely +cleans up that directory afterwards (including handling Windows-specific +read-only file permission issues). Downstream analysis stages in the +dependency_analyzer package operate on the local clone produced here. +""" + import os import shutil import tempfile diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c.py b/codewiki/src/be/dependency_analyzer/analyzers/c.py index 9332a6f8..5177dde2 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/c.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c.py @@ -1,3 +1,11 @@ +"""Tree-sitter based analyzer for C source files. + +This module parses C source code using tree-sitter to extract call-graph +nodes (functions, structs, and global variables) and the call/usage +relationships between them. It is part of the multi-language dependency +analysis pipeline, producing `Node` and `CallRelationship` objects that +feed into the broader dependency graph and clustering system. +""" import logging from typing import List, Optional, Tuple from pathlib import Path diff --git a/codewiki/src/be/dependency_analyzer/analyzers/typescript.py b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py index 2660e244..9edde854 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/typescript.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py @@ -210,6 +210,7 @@ def _get_parent_context(self, node) -> str: if node.parent.parent and node.parent.parent.type in ["module", "ambient_declaration"]: return "module_block" return "statement_block" + return "unknown" def _extract_function_entity(self, node, func_type: str, depth: int) -> dict: name_node = self._find_child_by_type(node, "identifier") if not name_node: diff --git a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index c49612c8..96d7abe7 100644 --- a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -1,3 +1,10 @@ +"""Dependency graph construction for repository analysis. + +This module defines the DependencyGraphBuilder class, which orchestrates +parsing a repository's source files, building a dependency graph from the +extracted components, validating graph completeness, and filtering leaf +nodes to those relevant for downstream processing. +""" from typing import Dict, List, Any import os from codewiki.src.config import Config @@ -154,4 +161,4 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: logger.info(f" ├─ Skipped (wrong type): {skipped_type}") logger.info(f" └─ Skipped (not found): {skipped_not_found}") - return components, keep_leaf_nodes \ No newline at end of file + return components, keep_leaf_nodes diff --git a/codewiki/src/be/dependency_analyzer/models/core.py b/codewiki/src/be/dependency_analyzer/models/core.py index 78f768fa..3902c374 100644 --- a/codewiki/src/be/dependency_analyzer/models/core.py +++ b/codewiki/src/be/dependency_analyzer/models/core.py @@ -1,3 +1,8 @@ +"""Core pydantic data models for the dependency analyzer. + +Defines the Node, CallRelationship, and Repository models that form the +core data contract used throughout the dependency analysis pipeline. +""" from pydantic import BaseModel from typing import List, Optional, Dict, Any, Set from datetime import datetime @@ -66,3 +71,4 @@ class Repository(BaseModel): clone_path: str analysis_id: str + diff --git a/codewiki/src/be/dependency_analyzer/utils/security.py b/codewiki/src/be/dependency_analyzer/utils/security.py index 724421e6..cabd76ae 100644 --- a/codewiki/src/be/dependency_analyzer/utils/security.py +++ b/codewiki/src/be/dependency_analyzer/utils/security.py @@ -1,3 +1,11 @@ +"""Security utilities for safe file access within a repository root. + +This module implements path-traversal and symlink protections used when +reading files from a repository. It ensures that file access is confined to +a given base directory and that symlinks are not followed, mitigating +directory-traversal and symlink-escape attacks during dependency analysis. +""" + from pathlib import Path import os @@ -31,3 +39,4 @@ def safe_open_text(base_dir: Path, target: Path, encoding="utf-8"): os.close(fd) except OSError: pass + diff --git a/codewiki/src/config.py b/codewiki/src/config.py index dfd58b4e..bcd89551 100644 --- a/codewiki/src/config.py +++ b/codewiki/src/config.py @@ -1,3 +1,19 @@ +"""Configuration module for CodeWiki. + +This module defines the central `Config` dataclass used throughout the +CodeWiki documentation-generation pipeline. It encapsulates repository +paths, output directories, LLM provider settings (models, API keys, base +URLs, temperatures, and token limits), and agent instruction customization +(include/exclude patterns, focus modules, doc type, custom instructions). + +It also provides constructors for building a `Config` instance from CLI +arguments (`from_args`), for a web-app job (`from_web_job`), from explicit +CLI parameters (`from_cli`), and from a `ConfigManager` +(`from_config_manager`), along with helpers for +multi-path source validation and prompt-addition generation used by the +downstream documentation generation stages. +""" + from dataclasses import dataclass, field, fields, asdict from typing import Optional, List, Dict, Any import argparse diff --git a/codewiki/src/utils.py b/codewiki/src/utils.py index 718e755c..19a0d50a 100644 --- a/codewiki/src/utils.py +++ b/codewiki/src/utils.py @@ -1,3 +1,5 @@ +"""File I/O utility helpers used across the CodeWiki backend and web app.""" + import os import json from typing import Any, Optional, Dict @@ -45,3 +47,4 @@ def load_text(filepath: str) -> str: return f.read() file_manager = FileManager() +