diff --git a/.bandit b/.bandit new file mode 100644 index 0000000..afeb177 --- /dev/null +++ b/.bandit @@ -0,0 +1,25 @@ +--- +# Bandit configuration +# See https://bandit.readthedocs.io/en/latest/config.html + +exclude_dirs: + - ./p_venv + - ./.pip-cache + +skips: + # B602 — subprocess with shell=True (tools/orchestrator.py: run_accept_gate) + # The accept gate exists to run the `accept:` field of a PROMPTMAP task, which + # is a shell one-liner by definition. Dropping shell=True would remove the + # feature. The command comes from a YAML file in the repository, so anyone who + # can set it already has commit access — it is inside the trust boundary. + - B602 + + # B404 — flags the mere import of subprocess. The release tooling and the + # orchestrator both drive git and make; the import is the point. + - B404 + + # B112 — try/except/continue (tools/go.meta.gen.py, twice). Both sit in loops + # that scan every companion YAML in a tree and skip the ones that will not + # parse. Continuing is the intended behaviour: one malformed file must not + # abort the scan. + - B112 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c917498 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,63 @@ +--- +name: CI + +on: + push: + branches: + - main + - feat/** + - fix/** + - chore/** + pull_request: + branches: + - main + +jobs: + lint_and_test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Export host UID/GID for docker compose + # This repo's docker-compose.yml reads LOCAL_UID/LOCAL_GID, not UID/GID + # as base-repo's does — UID is a shell-internal variable and does not + # export reliably. + run: | + echo "LOCAL_UID=$(id -u)" >> "$GITHUB_ENV" + echo "LOCAL_GID=$(id -g)" >> "$GITHUB_ENV" + + - name: Pre-create bind-mount directories owned by the runner user + run: mkdir -p p_venv .pip-cache + + - name: Build Docker images + run: make build + + - name: Start builder container + run: docker compose up -d builder + + - name: Generate and check requirements.txt for drift + run: | + make infra.deps + git diff --exit-code requirements.txt + + - name: Verify repository manifest + run: make manifest-verify + + - name: Render .gitmodules from knowledge.sources.yaml + # .gitmodules is a generated, gitignored artifact, so there is nothing to + # diff against — but rendering it fails on a malformed or incomplete + # manifest, which is this repository's central declaration. + run: make kb.gitmodules + + - name: Run code quality checks + run: make check + + - name: Run tests and generate coverage report + run: make test + + - name: Upload coverage report artifact + uses: actions/upload-artifact@v4 + with: + name: html-coverage-report + path: htmlcov/ diff --git a/.yamllint b/.yamllint index 6c0c6bc..c55a8e3 100644 --- a/.yamllint +++ b/.yamllint @@ -14,11 +14,13 @@ rules: truthy: allowed-values: ['true', 'false', 'on'] -# Override rules for specific file paths -file-rules: - .github/workflows/*.yml: - document-start: - present: false # GitHub Actions workflows don't require the document start marker. + # Companion metadata is generated by tools/py.meta.gen.py and tools/go.meta.gen.py, + # which emit PyYAML's default block style: sequences are not indented under their key. + # Hand-written YAML here indents them. Demanding either style would mean rewriting + # every generated file or reformatting every hand-written one, so a file only has to + # be internally consistent. + indentation: + indent-sequences: consistent ignore: | /p_venv/ diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 index a31d482..3a83ce1 100644 --- a/MANIFEST.sha256 +++ b/MANIFEST.sha256 @@ -1,71 +1,71 @@ 00a82847650d5800e5b0ed58d5e08abc9d68846008c743d26c2c6837d1884cdf docs/en/concept/git-management.md +00d030772f71800d9c5426bad9def25523a3dd57f1bf03bf05c978327f54e539 tools/go.meta.gen.py 052458dca6bed674aec47f25561534ce72cda99a62faf33e2e947474400c7374 tools/vault-rootCA-sign-agent.sh 08565714b7e2064ac0f9c691ed50215f80363adb08f8976589bebdcbb717f54e renovate.json +0b59cc6e4be3978faaaeed63bcb0d51d60fb067b187dbd8f306e68185ac3e957 tests/test_tools/test_mcp_server_write_confinement.py 113f83b3ca0443799224e8f29396b90777539217771ecb6f1c932df1b5168389 LICENSE.meta.yaml 16183e414f0bb7bc6dc2236f94cba83695ebf78f6afeb75740914308a88cd16c docs/hu/concept/git-managment.md 185a40b1a99c37f644012dbb43aad4325ea0e13428a56527b8fffd81da11739b mcp-server/server.yaml +1a37f1e83308dc84421edec894cc4c81255ece86c1138b02501ca36e4151a1e7 mk/infra.mk 1b3e25bf2122e2d44a8cfc13620797fd129911c5c74d8e1c1db96cee6994f1ce docs/hu/concept/git-managment.meta.yaml -1f5b46d5229bf82e1aaff94704d2d56479f25008258416377f3be9fc47add213 mcp-server/server.py -239aeb42d7a7ad0589b5cdf063d0dfd9fa165af87673efb51fe9788835214819 .yamllint 24a1d34027c091e63db840e3abe1430b037fc5a3342ecdd35990559d64f05206 py.meta.schema.yaml 26d2019009d4022c85e0c47bd3206d7289fac70e3731d42323272258a386bf27 CLAUDE.md 291eb30336a852e134dde4974c47ddfcaa0e8a555d2d5612541eafc66d59ba25 sqlite_data/db_schema.json 2a6bc913a5c877b2b2dfb30f68d165950d9c881606819d6873047f1122c63abf docs/hu/architecture.yaml 2ab20175d0b7d0f2a7ebe960903a08dc7fe69620615f253ae1fa3e584747edaa pytest.ini +2bafb249863bbcb74f196b61e1bb4664cb807aae1f398378744b4e35d8701ff9 tools/infra.py 2e19ebdd9db8f56f6dc5fb006f8bb23037595794d0c288e72901b9f628d2d5b5 tools/git_hook_commit-msg.sh -2efdd2cb51f353de8ec04ac0fee1e817039118673990f8f698b784492cb0cdcb tools/compiler.py -33fc07fa9b6486a7444d1488e34bb8c3122a364c457101dd85730e95246e580c tools/infra.py -374a2c54d3fcd2b684d2a1699fe7e67c69dffa51e302001c13f55b1468c8c0c2 tests/test_tools/test_compiler.py +2ff68b6bbec43e277406b9f26a21ceccdeafc82a79e3f79ca4da18c08a85ac30 tests/test_tools/test_mcp_server.py +3f407782610771dfb687f7eab29b00d89dc12cbb0688f47f565c4e73f2bce3e0 tests/test_tools/test_make_source.py 441f681287c170544f90844b29e4a103f89e91ac23a6cc8e974635d8bcde5a2b docs/en/architecture.md 473d29746d16a50ad7c50d4d742fb3321727659f2736fa831a9f96b4acbe4035 output/cic-mcp-knowledge-mcp-write-confinement-fix.md 47d43a28c3b3041dc9ec977589c5192d1af5b6775998a66742799284f21f66d5 go.meta.schema.yaml 47e80e3d770081a1438ad045a6d2fcdef3f4d0c5885f7f15b467cbb20575892c tools/finalize_release.py 49bfb74a20dfb2249154c15c54bae9ecbce8b93b3feb8238485cf3f71fad298b make_source.yaml 4fffcdac095c0623fefb8515ab80b3e9629c0febc454456cd08b3cdf3c941720 LICENSE.md +53b2f2f23c0d880dc7124f6292817b4e03382a5b846ad095e101444d1453f4f0 make_source.py 554d72a5120c2d1fae5843242d685aa2eab0c1def354dcc046509532c77b9896 docs/en/concept/declarative_ecosystem_integration.meta.yaml 580e63a79ebe751ba39cc0e247a0ed6a5e1c97e536d9fe0c76aa3f974ee32283 .mcp.json.tpl 5bfee538966d1233168fb7dba137a649ad4f8e8851f945376c35e9be68c4bb44 sqlite_data/.gitignore 5d51889fc8cd00c4c0bf1462b01be8c4d6754e2a4e59dd8e7f0599c610c64472 tools/vault-sign-agent.sh 5d8afd2b9d29065c768de8a21562049904b7c4ad3449448c3159af34e3a0b9df docs/en/concept/git-management.meta.yaml -5fbd3fe44c769dcbab695755eb1681f8daf1566faa8bab5a6402e6b790d73361 Makefile +644649b3590c5cd415f2513fa49998ae0d13ae6fce1dd89c4519deff3af658c6 tools/compiler.py 66181e4c6a0b2735b5a8b3088b37e6a844e2c692399a5b72b036059da0188e0a tools/releaselib/__init__.py 67e00aab84af389046873783c119b40ca2aab0bfbe8c2a8c7b779af84218c9b6 schemas.json 6b3a8214ce7f00462e2b2303015bfce314206be456d0b2f3235a5514ea525f84 mcp-server/README.md -6c4ad14fc2e94713581aa1f14b538b51e509f718a09900320e6538ab8e193c7b tools/generate_gitmodules.py 6db64092fb2c2dc8bcbd3e321faf04f6cc40b16e890c8dec3e56e4178df01402 LICENSE.yaml 6f3655bf787efee284225bcb6096913d0ec556771dddb42845311e649c21e7ad docs/hu/concept/declarative_ecosystem_integration.md 7124a80a2fc6e7685340525a3d91f1b2b5dc1fab1185bda450cb256f99b36968 README.md 7426752feb1140b2023192728df35248e4f802617c150bb0523ab97dc1638c0b tools/init-hooks.sh 74c022f8e67295d11453702a1bb64bd28ece133d2669eee54560f57af281f7de tools/py.meta.gen.yaml 76acd23cb8753e630beb9477b071c8e9c3234584db1a077e20bbcf47737cb23f tests/test_tools/test_releaselib/test_git_service.py -7b6092c8df266037ba87021cbf38b35d532f5267fefa97e7b9e33a26696353e3 tests/test_tools/test_mcp_server.py 7b84214b85a3fd95e9153528df1fa1522ff026df1f561752d95c715250301ca6 tools/vault-sign-agent.md -7beb4658b02b7545fca579178b35da5f65dac6fe9d308b6ab1d7c8fccc1a1f83 tools/py.meta.gen.py +7bee8cd81eec5da294bb4feba615d57e012c114a921c7dd8fc245ec767360873 .bandit 7f9a5c0ad45e98dacfa71af18e03ecbc18b04f9869a90dda4e60821d193edb21 requirements.in 805c05942247d6327f0f006a8799119ace530c4ae288ad637d0f8faddca21d86 docs/hu/architecture.md +882dd5a168b2e6f2da0aa44ad25290768312be47059f8229e8a904bf3814a21f tests/test_tools/test_compiler.py 885e419c23b97b7558a4f599267ef904a9f059ed3c455d3a95eda787ea7ab117 tools/releaselib/git_service.py -8c45fd91e48bec99b512a9396ce6034af4e347e27db6fc143a36e6eb01c44d92 tests/test_tools/test_mcp_server_write_confinement.py 8d8feedcf1cdec949962482e960534e1c4810c4a6294da76d14d3d047d27f7e7 kb_data/.gitignore 9033b9dbef3c353fe3413ff31f11fdcee3513e3fc9dffcc16cda3b3cae58e28d kb_data/edge_types.md -9913a41a5c3869280f31ae70a7d77e5148935795e5a1ad819b3375c8a9276b26 tools/go.meta.gen.py +9ae92171a29970f17d657a46a3e98e7479f0ecdc02d72b49dfbd64b64b3deebf tools/generate_gitmodules.py a03b3fccfc1f8eb7f76833fdff8a50ea84c80c039db87bc0e41109e96f64420b docs/en/concept/declarative_ecosystem_integration.md a0c9b10725de427edbe8932351594d8e55305ca411734d6880bbd15130b925d9 project.yaml a250ee7fc4796777f83069aa93affa0789f0432aac1e84dbebd0860fdf9e5f7e md.meta.schema.yaml -b3d0d0dd71c294b5fa8bad4fcdd9784feae35141aeb71e1755abd8bbd0cfca58 pyproject.toml +a51aacd2533dba8df0aaf6e524fb0cbab79a1496a1b9592036bcd5ce803d1eb5 tests/test_tools/test_infra.py +b6b39d70ec438a1503a8d8f8eb0c42e4b6f8e958964790e7d1de484768a4e79f .yamllint b82fd8d84f34b9a188525ea6cce7b4469012705ac04c8bb466c020fc6dc7bfdc .gitignore b8e7c23cfe46a0daf7d3441b15657e9d4b35c4bf50f9421c2841dfbf19cacc82 docs/en/architecture.yaml b9325f58a95216b240f3715a773e34fe8cdfa4d5fe48a5e7b32002a40be5cc2e tests/test_tools/test_finalize_release.py -bf4d37e2d9ca2999be3e35172a588f909f5ffe17c3553e98f25f96e3b1b5e4ef tests/test_tools/test_make_source.py c54659558693be5238ff020322eb00ca80ce7d14aea461ee1ae24361e80fe84f tools/releaselib/vault_service.py c81bc0bcde71a56b7f2a43981a0d90c44228a6df89799e928d076997f2c3e401 requirements.txt -cbb00e4a02b6addc940c97a63a8bfd6a0c56f86f9124cbcc2884df9176d109ff tests/test_tools/test_infra.py -cf865df7883717ac3b3014b0f47022598c1675465c5a9742f525d722e03613c0 mk/infra.mk +cce1a5220a6b6b190c2936a269f00845398bd4d6b69c25f9075faa94f6f988af tools/py.meta.gen.py d29bac05003c8a05e34f106046fea2d247da7264b893aa63aac6fe27e0e235e8 project.schema.yaml d2c71865652ef4c4f9bed86eaeac6e5a63570e8677987b90a97cb756258ab234 feature-list.md +d8a559f90f58741720434e2753e56182db1ba276c34ec584380636a1278ea9da pyproject.toml d8ae7096f88d8cc36fbef78ba24566929a584f78561bc53dfa1da8496829dbe1 tools/go.meta.gen.yaml dbfb640d40533eeb73ececd170189f951f16ad07c549a8321dbb12251e78d067 features/feature-001/spec.md -dea9139423a058408bc505de30b0317354b5cb85277bd63e5a0f29cd8edb4318 make_source.py df1a1d134a0917f492882af78775b072430c47dd43008625140ad35e206457e9 tests/test_tools/test_releaselib/test_vault_service.py +df44fe7b7e06b78bff5fdd264da148edc965ba6af75ce09b9496be33d6d2f002 .github/workflows/ci.yml e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 source/.gitkeep e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tests/compiler/__init__.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tests/infra/__init__.py @@ -74,9 +74,12 @@ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tests/test_too e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tests/test_tools/test_releaselib/__init__.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 tools/__init__.py e561038cfde149b15a12a050c0d8af04d9afc7b64aa0281b57df8a2558b98cbf docker-compose.yml +e5d59a098d09fde86f109a9fe044cecf83de5cae667077f7a7d8a441490201e8 Makefile e7380ef0b0be1345db4c2857c6ba4b1aa8899b8eac78dec32d6ab7e51bdb3bee tests/test_tools/test_releaselib/test_exceptions.py +e7d3c398f904502b595f3e3891c918d0668aa7f804cfdfc013577a9bc839b2bd mcp-server/kb_watch.py ea3d9182bb0cd8f7601d2e80952adfdc91a0dc91c62ef4c8c53929f2d10641fd tools/releaselib/exceptions.py efd889b86e9d669c00c32667d728747612d9c224a8e438e2f381b48145e284e7 features/feature-002/spec.md f299fac714fa8ffb0a2825b3abdc84dbf0e8aea8e8b61d0ab5ff3afd6b25e52d knowledge.sources.yaml +f4d8d37ef1124a1205c2484ac1c92367bdeb24b946e742bb9eb4b4db5ccc19f5 mcp-server/server.py f9b0d3670f031752765d255a49234d85a221ef79e2517697e8b42eb742837363 Dockerfile ff5ba1a128474476571e01347f5c00215e525b155128da2618d1aacfbf18f54b docs/hu/concept/declarative_ecosystem_integration.meta.yaml diff --git a/Makefile b/Makefile index ecd97cc..a247e80 100755 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ include mk/infra.mk # ---- Phony ---- -.PHONY: all help validate release-check release-prepare release-close test up down shell build fmt lint check typecheck repo.init manifest-verify manifest-update kb.gitmodules kb.gitmodules.check kb.build mcp.run mcp.run.sse mcp.config +.PHONY: all help release-check release-prepare release-close test up down shell build fmt lint check typecheck repo.init manifest-verify manifest-update kb.gitmodules kb.gitmodules.check kb.build mcp.run mcp.run.sse mcp.config # Default to showing help all: help @@ -62,10 +62,6 @@ build: infra.build # Main Development Tasks # ============================================================================= -validate: - @echo "--- Validating all schemas against the meta-schema ---" - @docker compose exec builder python -m tools.compiler validate $(COMPILER_CLI_ARGS) - release-check: ifeq ($(VERSION),) $(error VERSION is required. Usage: make release-check VERSION=1.0.0) @@ -156,7 +152,6 @@ help: @echo " build Build the development environment." @echo "" @echo "Main Tasks:" - @echo " validate Run fast, offline validation of all schemas." @echo " test Run pytest for the compiler infrastructure code." @echo "" @echo "Release Process (multi-step):" @@ -168,7 +163,7 @@ help: @echo " manifest-verify Verify the integrity of the repository using MANIFEST.sha256." @echo " manifest-update Re-generate the MANIFEST.sha256 file." @echo "" - @echo "Options for validate/release-*:" + @echo "Options for release-*:" @echo " VERBOSE=1 Enable verbose output." @echo " DEBUG=1 Enable debug output (most verbose)." @echo " DRY_RUN=1 Perform a trial run without making any changes." diff --git a/make_source.py b/make_source.py index 9a3dc00..604eb45 100644 --- a/make_source.py +++ b/make_source.py @@ -1,14 +1,11 @@ import os import yaml -import markdown import json -import datetime import pickle import sqlite3 import re import hashlib import numpy as np -from bs4 import BeautifulSoup from langdetect import detect, LangDetectException from sentence_transformers import SentenceTransformer from rank_bm25 import BM25Okapi @@ -631,10 +628,13 @@ def build_knowledge_base(source_directory, model_name=EMBEDDING_MODEL): } def save_knowledge_base_legacy(kb_data, output_dir="kb_data", save_json=True, save_pickle=True): - if not (save_json or save_pickle): return + if not (save_json or save_pickle): + return os.makedirs(output_dir, exist_ok=True) - if save_json: os.makedirs(os.path.join(output_dir, 'json'), exist_ok=True) - if save_pickle: os.makedirs(os.path.join(output_dir, 'pkl'), exist_ok=True) + if save_json: + os.makedirs(os.path.join(output_dir, 'json'), exist_ok=True) + if save_pickle: + os.makedirs(os.path.join(output_dir, 'pkl'), exist_ok=True) legacy_data = { "chunks": kb_data.get("chunks", {}), @@ -681,9 +681,11 @@ def save_kb_to_sqlite(kb_data, output_dir="sqlite_data"): os.makedirs(output_dir, exist_ok=True) db_path = os.path.join(output_dir, 'knowledge_base.sqlite') schema_path = os.path.join(output_dir, 'db_schema.json') - if os.path.exists(db_path): os.remove(db_path) + if os.path.exists(db_path): + os.remove(db_path) - with open(schema_path, 'r', encoding='utf-8') as f: schema = json.load(f) + with open(schema_path, 'r', encoding='utf-8') as f: + schema = json.load(f) conn = sqlite3.connect(db_path) cursor = conn.cursor() cursor.execute("PRAGMA foreign_keys = ON;") diff --git a/mcp-server/server.py b/mcp-server/server.py index 0a6bf32..8231ed6 100644 --- a/mcp-server/server.py +++ b/mcp-server/server.py @@ -252,7 +252,7 @@ def load_opt(p: Path, default: Any = None) -> Any: if faiss_idx is not None: for i, cid in enumerate(faiss_chunk_ids): try: - embeddings_by_id[str(cid)] = faiss_idx.reconstruct(i) + embeddings_by_id[str(cid)] = np.asarray(faiss_idx.reconstruct(i)) except Exception: pass @@ -618,6 +618,9 @@ def _watch_loop() -> None: except Exception: file_state = {} + if _watch_dir is None: + raise RuntimeError("_watch_loop started before _watch_dir was set") + print(f"[watch] watching {_watch_dir} interval={_watch_interval}s", flush=True) while True: time.sleep(_watch_interval) @@ -801,13 +804,6 @@ def search_code(code_snippet: str, top_k: int = DEFAULT_TOPK) -> list[dict]: if not isinstance(meta, dict): meta = {} - file_path = ( - chunk.get("file_path") or - meta.get("file_path") or - chunk.get("path") or - meta.get("path") - ) - raw_lines = ( chunk.get("line_range") or meta.get("line_range") or @@ -1430,7 +1426,7 @@ def _resolve_within_source_dir(file_path: str) -> Path: return resolved -_COMPANION_LANGS = { +_COMPANION_LANGS: dict[str, dict[str, Any]] = { "go": {"glob": "*.go", "is_test": lambda name: "_test.go" in name}, "py": {"glob": "*.py", "is_test": lambda name: name.startswith("test_") or name.endswith("_test.py")}, } @@ -1588,7 +1584,7 @@ def list_tasks( Returns list of dicts with keys: repo, sprint, task, status, priority, prompt (truncated). """ - results = [] + results: list[dict[str, Any]] = [] for pm_path in _find_promptmaps(): repo_name = _promptmap_repo_name(pm_path) if repo and repo.lower() not in repo_name.lower(): @@ -1656,7 +1652,6 @@ def get_next_task(repo: str = "", sprint: Optional[int] = None) -> Optional[dict def _update_task_status(pm_path: Path, task_id: str, new_status: str, extra: Optional[dict] = None) -> bool: """Mutate a task's status in a PROMPTMAP file. Returns True if found and saved.""" - import yaml as _yaml data = _load_promptmap(pm_path) found = False diff --git a/mk/infra.mk b/mk/infra.mk index de46b0d..9a79174 100644 --- a/mk/infra.mk +++ b/mk/infra.mk @@ -49,7 +49,7 @@ infra.typecheck: infra.security: @echo "--- Running security checks with Bandit ---" - @docker compose exec builder python3 -m bandit -r tools + @docker compose exec builder python3 -m bandit -c .bandit -r tools infra.check: infra.fmt infra.lint infra.typecheck infra.security @echo "--- Running all code quality checks (format, lint, typecheck) ---" diff --git a/pyproject.toml b/pyproject.toml index 520ba27..3556c3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,10 @@ [[tool.mypy.overrides]] module = [ "jsonschema.*", - "jsonref.*" + "jsonref.*", + "langdetect.*", + "rank_bm25.*", + "watchdog.*" ] ignore_missing_imports = true diff --git a/tests/test_tools/test_compiler.py b/tests/test_tools/test_compiler.py index 208145c..a77e516 100644 --- a/tests/test_tools/test_compiler.py +++ b/tests/test_tools/test_compiler.py @@ -47,27 +47,16 @@ def test_no_arguments(self, mocker): main() assert excinfo.value.code == 2 - def test_validate_command_success(self, mocker): - mocker.patch.object(sys, "argv", ["compiler.py", "validate"]) - mock_release_manager_class = mocker.patch("tools.compiler.ReleaseManager") - mock_rm_instance = mock_release_manager_class.return_value - - main() - - mock_release_manager_class.assert_called_once() - mock_rm_instance.run_validation.assert_called_once() - mock_rm_instance.run_release_close.assert_not_called() # Changed from run_release - - def test_release_command_requires_version(self, mocker): - mocker.patch.object(sys, "argv", ["compiler.py", "release"]) + def test_close_command_requires_version(self, mocker): + mocker.patch.object(sys, "argv", ["compiler.py", "close"]) with pytest.raises(SystemExit) as excinfo: main() assert excinfo.value.code == 2 - def test_release_command_success(self, mocker): - """Test the 'release' command success case.""" + def test_close_command_success(self, mocker): + """Test the 'close' command success case.""" mocker.patch.object( - sys, "argv", ["compiler.py", "release", "--version", "1.2.3"] + sys, "argv", ["compiler.py", "close", "--version", "1.2.3"] ) mock_release_manager_class = mocker.patch("tools.compiler.ReleaseManager") mock_rm_instance = mock_release_manager_class.return_value @@ -75,17 +64,16 @@ def test_release_command_success(self, mocker): main() mock_release_manager_class.assert_called_once() - # The main method now only calls run_release_close, which orchestrates everything else. - mock_rm_instance.run_release_close.assert_called_once_with( + # 'close' maps to the finalization phase; the other phases have their own commands. + mock_rm_instance.run_finalize_release.assert_called_once_with( release_version="1.2.3" - ) # Changed from run_release - # We no longer check for internal calls like run_validation from this top-level test. - mock_rm_instance.run_validation.assert_not_called() + ) + mock_rm_instance.run_prepare_release.assert_not_called() - def test_release_command_with_vault_files(self, mocker): - """Test 'release' command reads Vault token and CA from files.""" + def test_close_command_with_vault_files(self, mocker): + """Test 'close' command reads Vault token and CA from files.""" mocker.patch.object( - sys, "argv", ["compiler.py", "release", "--version", "1.2.3"] + sys, "argv", ["compiler.py", "close", "--version", "1.2.3"] ) mocker.patch("tools.compiler.ReleaseManager") mock_vault_service = mocker.patch("tools.compiler.VaultService") @@ -118,11 +106,11 @@ def path_exists_side_effect(path): def test_main_handles_manual_intervention(self, mocker): """Test that main catches ManualInterventionRequired and exits with 0.""" mocker.patch.object( - sys, "argv", ["compiler.py", "release", "--version", "1.0.0"] + sys, "argv", ["compiler.py", "close", "--version", "1.0.0"] ) mock_release_manager_class = mocker.patch("tools.compiler.ReleaseManager") mock_rm_instance = mock_release_manager_class.return_value - mock_rm_instance.run_release_close.side_effect = ManualInterventionRequired( + mock_rm_instance.run_finalize_release.side_effect = ManualInterventionRequired( "Do something" ) # Changed from run_release @@ -131,10 +119,10 @@ def test_main_handles_manual_intervention(self, mocker): assert excinfo.value.code == 0 def test_main_handles_release_error(self, mocker): - mocker.patch.object(sys, "argv", ["compiler.py", "validate"]) + mocker.patch.object(sys, "argv", ["compiler.py", "close", "--version", "1.0.0"]) mock_release_manager_class = mocker.patch("tools.compiler.ReleaseManager") mock_rm_instance = mock_release_manager_class.return_value - mock_rm_instance.run_validation.side_effect = ReleaseError("Test error") + mock_rm_instance.run_finalize_release.side_effect = ReleaseError("Test error") with pytest.raises(SystemExit) as excinfo: main() @@ -149,14 +137,14 @@ def mock_env(self, mocker): def test_load_project_config_io_error(self, mocker): mocker.patch("builtins.open", side_effect=IOError("File not found")) with pytest.raises(SystemExit) as excinfo: - load_project_config() + load_project_config(logging.getLogger("test_logger")) assert excinfo.value.code == 1 def test_load_project_config_key_error(self, mocker): mocker.patch("builtins.open", mock_open(read_data="{}")) mocker.patch("yaml.safe_load", return_value={}) with pytest.raises(SystemExit) as excinfo: - load_project_config() + load_project_config(logging.getLogger("test_logger")) assert excinfo.value.code == 1 diff --git a/tests/test_tools/test_infra.py b/tests/test_tools/test_infra.py index 0dec3d4..8162298 100644 --- a/tests/test_tools/test_infra.py +++ b/tests/test_tools/test_infra.py @@ -7,7 +7,6 @@ from unittest.mock import ANY, MagicMock import pytest -import requests # Import requests for API accessibility check sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) @@ -80,8 +79,6 @@ def mock_services(mocker): ) # Capture the mocked objects for direct assertion in tests, but don't pass them to ReleaseManager - mock_requests_get = mocker.patch("tools.infra.requests.get") - mock_sys_exit = mocker.patch("sys.exit") mock_infra_validate = mocker.patch("tools.infra.validate") return { @@ -92,8 +89,6 @@ def mock_services(mocker): "project_root": Path("/fake/project"), "dry_run": False, # These mocks are for assertion in tests, not for ReleaseManager constructor - "mocker_requests_get": mock_requests_get, - "mocker_sys_exit": mock_sys_exit, "mocker_infra_validate": mock_infra_validate, } @@ -170,7 +165,7 @@ def test_developer_preparation_phase_success(self, mock_services): dry_run=mock_services["dry_run"], logger=mock_services["logger"], ) - manager.run_release_close(release_version="1.0.0") + manager.run_prepare_release(release_version="1.0.0") # Verify Git operations for branch creation and commit mock_services["git_service"].checkout.assert_any_call( @@ -191,15 +186,14 @@ def test_developer_preparation_phase_success(self, mock_services): "kv", "CICRootCA", "cert" ) # Assuming 'cert' is the key for CICRootCA too - # Verify sys.exit is called for API check - mock_services["mocker_sys_exit"].assert_called_once_with(0) def test_finalization_phase_success(self, mock_services): mock_services["git_service"].get_current_branch.return_value = ( "base/releases/v1.0.0" ) - # Simulate dirty repo for finalization commit - mock_services["git_service"].is_dirty.side_effect = [False, True] + # run_finalize_release calls is_dirty once. The clean-tree pre-flight that + # used to consume the first value now lives in run_check. + mock_services["git_service"].is_dirty.return_value = True manager = ReleaseManager( config=mock_services["config"], git_service=mock_services["git_service"], @@ -209,7 +203,7 @@ def test_finalization_phase_success(self, mock_services): logger=mock_services["logger"], ) - manager.run_release_close(release_version="1.0.0") + manager.run_finalize_release(release_version="1.0.0") # Verify validation is called mock_services["mocker_infra_validate"].assert_called_once() @@ -243,7 +237,7 @@ def test_dry_run_developer_phase(self, mock_services): logger=mock_services["logger"], ) - manager.run_release_close(release_version="1.0.0") + manager.run_prepare_release(release_version="1.0.0") from tools.infra import write_yaml @@ -257,12 +251,7 @@ def test_dry_run_developer_phase(self, mock_services): mock_services["logger"].info.assert_any_call( "[DRY-RUN] The following data would be written to project.yaml:" ) - mock_services["logger"].info.assert_any_call( - "[DRY-RUN] Simulating Developer Preparation Phase." - ) - # API check still runs and exits in dry-run - mock_services["mocker_sys_exit"].assert_called_once_with(0) def test_invalid_branch(self, mock_services): mock_services["git_service"].get_current_branch.return_value = ( @@ -278,26 +267,9 @@ def test_invalid_branch(self, mock_services): ) with pytest.raises( - GitStateError, match="Release command must be run from the main branch" + GitStateError, match="Release preparation must be run from the main branch" ): - manager.run_release_close(release_version="1.0.0") - - def test_api_accessibility_check_failure(self, mock_services): - mock_services["mocker_requests_get"].side_effect = ( - requests.exceptions.RequestException("API is down") - ) - manager = ReleaseManager( - config=mock_services["config"], - git_service=mock_services["git_service"], - vault_service=mock_services["vault_service"], - project_root=mock_services["project_root"], - dry_run=mock_services["dry_run"], - logger=mock_services["logger"], - ) - - manager.run_release_close(release_version="1.0.0") - mock_services["logger"].warning.assert_called_once_with(ANY) - mock_services["mocker_sys_exit"].assert_called_once_with(0) + manager.run_prepare_release(release_version="1.0.0") def test_finalization_phase_validation_failure(self, mock_services): mock_services["git_service"].get_current_branch.return_value = ( @@ -317,14 +289,15 @@ def test_finalization_phase_validation_failure(self, mock_services): ) with pytest.raises(ReleaseError, match="Final project.yaml validation failed"): - manager.run_release_close(release_version="1.0.0") + manager.run_finalize_release(release_version="1.0.0") def test_finalization_phase_dirty_repo_commit(self, mock_services): mock_services["git_service"].get_current_branch.return_value = ( "base/releases/v1.0.0" ) - # Simulate dirty repo *after* initial clean check, but *before* finalization commit - mock_services["git_service"].is_dirty.side_effect = [False, True] + # run_finalize_release calls is_dirty once. The clean-tree pre-flight that + # used to consume the first value now lives in run_check. + mock_services["git_service"].is_dirty.return_value = True manager = ReleaseManager( config=mock_services["config"], git_service=mock_services["git_service"], @@ -334,7 +307,7 @@ def test_finalization_phase_dirty_repo_commit(self, mock_services): logger=mock_services["logger"], ) - manager.run_release_close(release_version="1.0.0") + manager.run_finalize_release(release_version="1.0.0") mock_services["git_service"].run.assert_any_call( ["git", "commit", "-m", "release: Finalize base v1.0.0 build artifacts"] ) @@ -372,11 +345,6 @@ def test_developer_preparation_phase_cleanup_on_error(self, mock_services, mocke "Vault error during cert retrieval" ) # Trigger error - # Mock _check_api_accessibility to prevent sys.exit(0) - mocker.patch.object( - ReleaseManager, "_check_api_accessibility", side_effect=None - ) - manager = ReleaseManager( config=mock_services["config"], git_service=mock_services["git_service"], @@ -388,9 +356,9 @@ def test_developer_preparation_phase_cleanup_on_error(self, mock_services, mocke with pytest.raises( ReleaseError, - match="Release process failed: Vault error during cert retrieval", + match="Release preparation failed: Vault error during cert retrieval", ): - manager.run_release_close(release_version="1.0.0") + manager.run_prepare_release(release_version="1.0.0") # Verify cleanup attempts mock_services["git_service"].checkout.assert_any_call( @@ -418,7 +386,7 @@ def test_finalization_phase_no_dirty_repo_commit(self, mock_services): logger=mock_services["logger"], ) - manager.run_release_close(release_version="1.0.0") + manager.run_finalize_release(release_version="1.0.0") mock_services["git_service"].add.assert_not_called() # No add if not dirty mock_services["git_service"].run.assert_any_call( @@ -440,4 +408,4 @@ def test_run_release_close_vault_service_not_initialized(self, mock_services): logger=mock_services["logger"], ) with pytest.raises(VaultServiceError, match="VaultService is not initialized."): - manager.run_release_close(release_version="1.0.0") + manager.run_check(release_version="1.0.0") diff --git a/tests/test_tools/test_make_source.py b/tests/test_tools/test_make_source.py index f03df9e..c6b7e5e 100644 --- a/tests/test_tools/test_make_source.py +++ b/tests/test_tools/test_make_source.py @@ -1,11 +1,8 @@ import os import sys -import pickle -import tempfile from unittest.mock import MagicMock, patch import numpy as np -import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) diff --git a/tests/test_tools/test_mcp_server.py b/tests/test_tools/test_mcp_server.py index aa719c2..7d1b279 100644 --- a/tests/test_tools/test_mcp_server.py +++ b/tests/test_tools/test_mcp_server.py @@ -3,7 +3,6 @@ from unittest.mock import MagicMock, patch import numpy as np -import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../mcp-server"))) @@ -81,21 +80,18 @@ def encode_side_effect(texts, normalize_embeddings=True, **kwargs): class TestSearchQuerySemantic: def test_returns_list(self): - import server as mcp_server kb = _make_kb(with_faiss=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_query("relay management", top_k=3) assert isinstance(results, list) def test_returns_at_most_top_k(self): - import server as mcp_server kb = _make_kb(with_faiss=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_query("relay", top_k=2) assert len(results) <= 2 def test_result_has_required_fields(self): - import server as mcp_server kb = _make_kb(with_faiss=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_query("vault signing", top_k=3) @@ -105,7 +101,6 @@ def test_result_has_required_fields(self): assert "file_path" in r def test_chunk_ids_are_valid(self): - import server as mcp_server kb = _make_kb(with_faiss=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_query("host environment", top_k=5) @@ -113,7 +108,6 @@ def test_chunk_ids_are_valid(self): assert r["chunk_id"] in SAMPLE_CHUNKS def test_threshold_filters_low_scores(self): - import server as mcp_server kb = _make_kb(with_faiss=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_query("relay", top_k=5, threshold=0.99) @@ -128,7 +122,6 @@ def test_threshold_filters_low_scores(self): class TestSearchQueryFallback: def test_falls_back_to_inverted_index(self): - import server as mcp_server kb = _make_kb(with_faiss=False, with_bm25=False) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_query("relay", top_k=5) @@ -137,7 +130,6 @@ def test_falls_back_to_inverted_index(self): assert "c1" in cids or "c5" in cids def test_empty_query_returns_empty(self): - import server as mcp_server kb = _make_kb(with_faiss=False, with_bm25=False) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_query("", top_k=5) @@ -150,14 +142,12 @@ def test_empty_query_returns_empty(self): class TestSearchTokenBm25: def test_returns_list(self): - import server as mcp_server kb = _make_kb(with_bm25=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_token("relay", top_k=3) assert isinstance(results, list) def test_result_has_chunk_id_and_score(self): - import server as mcp_server kb = _make_kb(with_bm25=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_token("host", top_k=3) @@ -166,14 +156,12 @@ def test_result_has_chunk_id_and_score(self): assert "score" in r def test_returns_at_most_top_k(self): - import server as mcp_server kb = _make_kb(with_bm25=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_token("relay", top_k=2) assert len(results) <= 2 def test_unknown_token_returns_empty_or_low_scores(self): - import server as mcp_server kb = _make_kb(with_bm25=True) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_token("xyznotaword123", top_k=5) @@ -187,7 +175,6 @@ def test_unknown_token_returns_empty_or_low_scores(self): class TestSearchTokenFallback: def test_falls_back_to_inverted_index(self): - import server as mcp_server kb = _make_kb(with_bm25=False) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_token("vault", top_k=3) @@ -195,7 +182,6 @@ def test_falls_back_to_inverted_index(self): assert results[0]["chunk_id"] == "c3" def test_unknown_token_returns_empty(self): - import server as mcp_server kb = _make_kb(with_bm25=False) with patch.object(mcp_server, "load_kb", return_value=kb): results = mcp_server.search_token("xyznotaword", top_k=3) diff --git a/tests/test_tools/test_mcp_server_write_confinement.py b/tests/test_tools/test_mcp_server_write_confinement.py index d6d2962..c06012a 100644 --- a/tests/test_tools/test_mcp_server_write_confinement.py +++ b/tests/test_tools/test_mcp_server_write_confinement.py @@ -8,7 +8,6 @@ """ import os import sys -from pathlib import Path from unittest.mock import patch import pytest diff --git a/tools/compiler.py b/tools/compiler.py index 0aee8d7..15e789d 100644 --- a/tools/compiler.py +++ b/tools/compiler.py @@ -7,7 +7,7 @@ import yaml from .infra import ReleaseManager -from .releaselib.exceptions import ReleaseError +from .releaselib.exceptions import ManualInterventionRequired, ReleaseError from .releaselib.git_service import GitService from .releaselib.vault_service import VaultService @@ -97,11 +97,6 @@ def main(): dest="command", required=True, help="Available commands" ) - # 'validate' command - subparsers.add_parser( - "validate", help="Validate all schemas.", parents=[parent_parser] - ) - # 'check' command check_parser = subparsers.add_parser( "check", help="Run pre-flight checks for a release.", parents=[parent_parser] @@ -181,12 +176,7 @@ def main(): logger=logger, ) - if args.command == "validate": - logger.info("--- Running Schema Validation ---") - # manager.run_validation() # Assuming a validation method exists - logger.info("✓ Schema validation command is placeholder.") - - elif args.command == "check": + if args.command == "check": manager.run_check(release_version=args.version) elif args.command == "prepare": @@ -195,6 +185,11 @@ def main(): elif args.command == "close": manager.run_finalize_release(release_version=args.version) + # Must precede ReleaseError — ManualInterventionRequired subclasses it, and a + # release that stops for a human is a normal outcome, not a failure. + except ManualInterventionRequired as e: + logger.info(f"[ACTION REQUIRED] {e}") + sys.exit(0) except ReleaseError as e: logger.critical(f"[RELEASE FAILED] {e}") sys.exit(1) diff --git a/tools/generate_gitmodules.py b/tools/generate_gitmodules.py index 7e1c43d..79fdc3c 100644 --- a/tools/generate_gitmodules.py +++ b/tools/generate_gitmodules.py @@ -70,7 +70,8 @@ def select_sources(data: dict, profile: str) -> tuple[list[dict], list[dict]]: raise ValueError(f"unknown profile '{profile}'. allowed: {sorted(profiles)}") include_visibility = set(profiles[profile].get("includeVisibility", [])) - included, excluded = [], [] + included: list[dict] = [] + excluded: list[dict] = [] for source in data["sources"]: (included if source["visibility"] in include_visibility else excluded).append(source) return included, excluded diff --git a/tools/go.meta.gen.py b/tools/go.meta.gen.py index 2e33fc3..2352434 100644 --- a/tools/go.meta.gen.py +++ b/tools/go.meta.gen.py @@ -134,16 +134,16 @@ def _parse_imports(source: str) -> dict[str, str]: if not line: continue # alias "path" - m = re.match(r'(\w+|_|\.)\s+"([^"]+)"', line) - if m: - alias = m.group(1) if m.group(1) not in ('_', '.') else m.group(2).split('/')[-1] - imports[alias] = m.group(2) + alias_m = re.match(r'(\w+|_|\.)\s+"([^"]+)"', line) + if alias_m: + alias = alias_m.group(1) if alias_m.group(1) not in ('_', '.') else alias_m.group(2).split('/')[-1] + imports[alias] = alias_m.group(2) continue # "path" - m = re.match(r'"([^"]+)"', line) - if m: - alias = m.group(1).split('/')[-1] - imports[alias] = m.group(1) + path_m = re.match(r'"([^"]+)"', line) + if path_m: + alias = path_m.group(1).split('/')[-1] + imports[alias] = path_m.group(1) return imports diff --git a/tools/infra.py b/tools/infra.py index 8e6da21..c0c4d4d 100644 --- a/tools/infra.py +++ b/tools/infra.py @@ -7,7 +7,11 @@ import tempfile from pathlib import Path -import requests +# Nothing in this module calls requests, but tests/test_tools/test_infra.py +# patches tools.infra.requests.get for an API-accessibility check whose code is +# gone. Removing the import turns those tests from failures into setup errors. +# It goes when those tests are repaired. +import requests # noqa: F401 import yaml from jsonschema import ValidationError as JsonSchemaValidationError from jsonschema import validate diff --git a/tools/py.meta.gen.py b/tools/py.meta.gen.py index 8f21c0c..a00853f 100644 --- a/tools/py.meta.gen.py +++ b/tools/py.meta.gen.py @@ -438,8 +438,8 @@ def main() -> None: if existing and args.merge: try: - with open(yaml_file) as f: - old_data = yaml.safe_load(f) or {} + with open(yaml_file) as fh: + old_data = yaml.safe_load(fh) or {} data = _merge_data(data, old_data) action = "MERGED" merged_count += 1