diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1be5786 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# scripts/check_no_client_leak.py is copied byte-for-byte from the canonical source +# (Statyx/Fab-Analyze-Data-Agent). Its SHA-256 is the proof it was not edited, so it must +# keep LF endings on every platform — core.autocrlf=true would otherwise rewrite it to CRLF +# on checkout and break the hash for anyone verifying it on Windows. +scripts/check_no_client_leak.py text eol=lf +scripts/check_client_denylist.py text eol=lf diff --git a/.github/workflows/no-client-leak.yml b/.github/workflows/no-client-leak.yml index 8f3d023..a549944 100644 --- a/.github/workflows/no-client-leak.yml +++ b/.github/workflows/no-client-leak.yml @@ -13,36 +13,24 @@ jobs: steps: - uses: actions/checkout@v4 - # This repo is public. Any real customer name or hardcoded Entra/Fabric identifier - # is a leak, so the build fails loudly instead of quietly shipping it. - # The names below are written with a single letter parenthesised (ERE groups match the - # same string) so that this guard file is not itself a hit for its own scan. - - name: Scan for client names - run: | - PATTERN='p(u)blicis|v(i)vatech|o(r)ange|a(i)r.?liquide|t(e)chnip|f(o)rvia|v(e)olia|a(r)kema|c(a)pgemini|e(i)ffage|o(p)mobility' - if grep -rniE "$PATTERN" \ - --exclude-dir=.git \ - --exclude-dir=node_modules \ - --exclude-dir=__pycache__ \ - . ; then - echo "::error::A known client name appears in the repository. Anonymise it before merging." - exit 1 - fi - echo "OK - no client name found." + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - - name: Scan for hardcoded clientId / tenantId GUIDs - run: | - PATTERN='(client_?id|tenant_?id)["'"'"']?[[:space:]]*[:=][[:space:]]*["'"'"'][0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}' - if grep -rniE "$PATTERN" \ - --exclude-dir=.git \ - --exclude-dir=node_modules \ - --exclude-dir=__pycache__ \ - --exclude-dir=workflows \ - . ; then - echo "::error::A clientId/tenantId GUID is hardcoded. Inject it from the environment instead." - exit 1 - fi - echo "OK - no hardcoded clientId/tenantId." + # This repo is public. Detection is by SHAPE, never by name: a list of customer + # names committed here would itself be the leak it is meant to prevent, and + # obfuscating that list only fools full-text search, not a reader. + # scripts/check_no_client_leak.py is copied byte-for-byte from the canonical + # source (Statyx/Fab-Analyze-Data-Agent@6e4a65e7) — do not edit it locally. + - name: Structural scan (GUIDs, Fabric endpoints, personal paths) + run: python scripts/check_no_client_leak.py + + # Name-based detection is preserved, but the names live in a secret, never in + # the source tree. Secret missing (forks, external PRs) -> warning, still green. + - name: Client deny-list scan (from secret, optional) + env: + CLIENT_DENYLIST: ${{ secrets.CLIENT_DENYLIST }} + run: python scripts/check_client_denylist.py tests: name: pytest diff --git a/.gitignore b/.gitignore index 998b4ff..a1ed0ac 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,11 @@ _*.json Thumbs.db *.swp +# Local client deny-list read by scripts/check_client_denylist.py. It holds real customer +# names, so it must never be committed to this public repo — in CI the same list comes from +# the CLIENT_DENYLIST secret instead. +.clientdeny + # Environment-specific — contain tenant / workspace / item IDs. Use the *.example files as templates. src/config.yaml src/state.json diff --git a/README.md b/README.md index 9c0d8a2..44d881f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ![Microsoft Fabric](https://img.shields.io/badge/Microsoft-Fabric-0078D4?logo=microsoft&logoColor=white) ![Fabric IQ](https://img.shields.io/badge/Fabric_IQ-Ontology_+_Graph-6242C9) ![Python](https://img.shields.io/badge/Python-3.12-3776AB?logo=python&logoColor=white) -![Tests](https://img.shields.io/badge/tests-39_passing-2E9E44) +![Tests](https://img.shields.io/badge/tests-50_passing-2E9E44) ![Portal](https://img.shields.io/badge/Portal-FastAPI-009688?logo=fastapi&logoColor=white) ![Deploy](https://img.shields.io/badge/deploy-idempotent-896610) @@ -77,7 +77,7 @@ flowchart LR copy src\config.example.yaml src\config.yaml # 2. ✅ Mandatory test gate — never deploy on red -python -m pytest tests/ -v --tb=short # 39/39 offline +python -m pytest tests/ -v --tb=short # 50/50 offline # 3. Deploy everything (idempotent — generates data, then ends with a warm-up) python src\deploy_all.py @@ -194,6 +194,7 @@ node build_deck.js # → presentation/Architecture_Vision.pptx - ✅ **Test gate is mandatory** — `python -m pytest tests/ -v` before any `deploy_*.py`. On red: **stop, fix the code**. - ♻️ **Idempotent deploys** — every step reads/writes `src/state.json`; re-running is safe. - 🔐 **Secrets never committed** — `src/config.yaml`, `src/state.json`, `.vscode/mcp.json` are gitignored (use the `*.example` templates). No tenant / workspace / app-registration ID is hardcoded anywhere; CI enforces this. +- 🕵️ **Leak detection is by shape, not by name** — `scripts/check_no_client_leak.py` (copied byte-for-byte from the canonical source, do not edit it) flags real GUIDs, real Fabric SQL endpoints and personal paths in Git-tracked files. Customer names are matched separately by `scripts/check_client_denylist.py`, whose list comes from the `CLIENT_DENYLIST` repository secret (one name per line) or a gitignored local `.clientdeny`. **The names are never committed to this public repo** — that list would be the leak. No secret configured: the rule warns and passes. - 🧪 **Synthetic data only** — every organisation, venue and sponsor in this repo is fictional. - ⚡ **F2+ capacity** — resume the capacity before a deploy/demo (Ontology / Graph / agents need it). - 🧠 **Reports are Legacy PBIX** — see the report gotchas below. diff --git a/scripts/check_client_denylist.py b/scripts/check_client_denylist.py new file mode 100644 index 0000000..686908c --- /dev/null +++ b/scripts/check_client_denylist.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Fail if a name from the private client deny-list appears in Git-tracked files. + +This repo is public, so the deny-list itself must never be committed here. The +list is injected at run time and stays out of the source tree: + + 1. ``CLIENT_DENYLIST`` -- one entry per line (GitHub Actions secret). + 2. ``.clientdeny`` at the repo root -- gitignored, for local runs. + +Neither present? The rule is skipped with a warning and the build stays green. +An absent secret must never break a fork, a PR from outside the org, or a +contributor's laptop. + +Only files returned by ``git ls-files`` are scanned, for the same reason as the +structural checker: ``__pycache__/*.pyc`` and ``node_modules`` embed absolute +paths and produce false positives. + +Two deliberate properties: + +* **Nothing is echoed.** A finding prints the file, the line number and a hash + prefix of the entry -- never the entry itself, never the source line. CI logs + of a public repo are public: a chatty scanner would re-publish the very name + it is guarding. +* **Colour contexts are ignored.** Some client names double as ordinary colour + words. A hit on a line that is plainly about colour -- a shields.io badge, a + hex token, ``rgb()``, a CSS declaration -- is dropped. The rule is generic; no + name and no colour is hardcoded here. + +Usage: + python scripts/check_client_denylist.py + +Exit code 0 = clean or not configured, 1 = a deny-listed name was found. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +LOCAL_DENYLIST = REPO_ROOT / ".clientdeny" + +BINARY_SUFFIXES = { + ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".pptx", ".xlsx", + ".docx", ".zip", ".gz", ".woff", ".woff2", ".ttf", ".eot", +} + +# A line that is plainly about colour. Generic on purpose: it keys on the +# colour syntax around the hit, not on which word was matched. +COLOUR_CONTEXT_RE = re.compile( + r""" + img\.shields\.io # badge URLs: .../badge/x-y- + | \#[0-9A-Fa-f]{3,8}\b # hex token + | \brgba?\s*\( # rgb() / rgba() + | \bhsla?\s*\( # hsl() / hsla() + | (?:^|[;{,\s])(?:color|background|background-color|fill|stroke|border + |border-color|outline|box-shadow|--[\w-]+)\s*: # CSS declaration + | "(?:color|colour|fill|stroke|background)"\s*: # JSON theme key + """, + re.IGNORECASE | re.VERBOSE, +) + +# The obfuscation this whole change exists to remove: a single character wrapped +# in parentheses. `n(a)me` still matches as an ERE and reads fine to a human, but +# no full-text search for the name finds it. Parentheses are stripped before +# matching so the trick buys nothing. +SINGLE_CHAR_PAREN_RE = re.compile(r"\(([A-Za-z0-9])\)") + + +def load_denylist() -> tuple[list[str], str]: + """Return (entries, source-label). Empty list means "not configured".""" + raw = os.environ.get("CLIENT_DENYLIST", "") + source = "the CLIENT_DENYLIST environment variable" + if not raw.strip() and LOCAL_DENYLIST.is_file(): + raw = LOCAL_DENYLIST.read_text(encoding="utf-8", errors="ignore") + source = str(LOCAL_DENYLIST.name) + + entries = [] + for line in raw.splitlines(): + entry = line.strip() + if entry and not entry.startswith("#"): + entries.append(entry) + return entries, source + + +def compile_entry(entry: str) -> re.Pattern[str]: + """Word-bounded, case-insensitive, separator-tolerant literal match. + + A two-word entry also catches the hyphenated, underscored and glued + spellings -- the separator is where obfuscation is cheapest. + """ + parts = [re.escape(part) for part in entry.split()] + body = r"[\s._\-]*".join(parts) + return re.compile(rf"\b{body}\b", re.IGNORECASE) + + +def fingerprint(entry: str) -> str: + """Stable, non-reversible label so a maintainer can tell entries apart.""" + digest = hashlib.sha256(entry.strip().lower().encode("utf-8")).hexdigest() + return digest[:8] + + +def tracked_files() -> list[str]: + out = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + return [p for p in out.split("\0") if p] + + +def scan_file(rel_path: str, patterns: list[tuple[int, str, re.Pattern[str]]]) -> list[str]: + path = REPO_ROOT / rel_path + if path.suffix.lower() in BINARY_SUFFIXES or not path.is_file(): + return [] + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return [] + + findings = [] + for lineno, line in enumerate(text.splitlines(), start=1): + colour_line = COLOUR_CONTEXT_RE.search(line) is not None + if colour_line: + continue # the word is being used as a colour, not as a name + variants = {line, SINGLE_CHAR_PAREN_RE.sub(r"\1", line)} + for index, mark, pattern in patterns: + if any(pattern.search(variant) for variant in variants): + findings.append(f"{rel_path}:{lineno}: deny-list entry #{index} [{mark}]") + return findings + + +def main() -> int: + entries, source = load_denylist() + if not entries: + print( + "::warning::Client deny-list not configured " + "(set the CLIENT_DENYLIST secret, or a gitignored .clientdeny file) " + "- name-based scanning skipped. Structural checks still ran." + ) + return 0 + + patterns = [ + (index, fingerprint(entry), compile_entry(entry)) + for index, entry in enumerate(entries, start=1) + ] + + findings: list[str] = [] + for rel_path in tracked_files(): + findings.extend(scan_file(rel_path, patterns)) + + if findings: + # Entries are reported by index + hash prefix only: printing the name + # here would leak it into public CI logs. + print( + f"Deny-listed client name(s) found in Git-tracked files " + f"({len(entries)} entries loaded from {source}):\n" + ) + for finding in findings: + print(f" {finding}") + print( + "\nThe matched text is deliberately not shown. Open the file at the " + "line above and anonymise it before merging." + ) + return 1 + + print(f"No deny-listed client name found ({len(entries)} entries checked).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_no_client_leak.py b/scripts/check_no_client_leak.py new file mode 100644 index 0000000..bf1b825 --- /dev/null +++ b/scripts/check_no_client_leak.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Fail if real customer / tenant identifiers leak into Git-tracked files. + +This repo is public. It must never contain real Microsoft Fabric identifiers, +real workspace SQL endpoints, or personal filesystem paths. + +Only files returned by ``git ls-files`` are scanned. The working tree is +deliberately NOT walked: ``__pycache__/*.pyc`` and ``node_modules`` embed +absolute build paths and would produce false positives. + +Usage: + python scripts/check_no_client_leak.py + +Exit code 0 = clean, 1 = leak detected. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# Placeholder GUIDs that are allowed to appear verbatim. +ALLOWED_GUIDS = { + "00000000-0000-0000-0000-000000000000", +} + +BINARY_SUFFIXES = { + ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".pptx", ".xlsx", + ".docx", ".zip", ".gz", ".woff", ".woff2", ".ttf", ".eot", +} + +GUID_RE = re.compile( + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" +) + +# A real Fabric SQL/warehouse endpoint host: a long opaque token followed by +# ".datawarehouse.fabric.microsoft.com" (or the Synapse / Power BI equivalents). +FABRIC_ENDPOINT_RE = re.compile( + r"\b[a-z0-9]{20,}(?:-[a-z0-9]{20,})?\." + r"(?:datawarehouse|dev\.azuresynapse|pbidedicated)\b", + re.IGNORECASE, +) + +# Personal home directories. Placeholders such as C:\Users\ are fine. +PERSONAL_PATH_RE = re.compile( + r"(?:[A-Za-z]:\\Users\\|/Users/|/home/)(?!<|\{|\$|%|USERNAME\b|username\b|you\b|user\b)" + r"[A-Za-z0-9._-]+", +) + +# Extra literals to forbid, as a comma-separated list. Deliberately empty by +# default: hardcoding an author's username here would re-publish the very thing +# this checker exists to remove. +# NO_LEAK_EXTRA_LITERALS="acme,contoso" python scripts/check_no_client_leak.py +FORBIDDEN_LITERALS = [ + literal.strip().lower() + for literal in os.environ.get("NO_LEAK_EXTRA_LITERALS", "").split(",") + if literal.strip() +] + +CHECKS = [ + ("real GUID", GUID_RE, lambda m: m.group(0).lower() not in ALLOWED_GUIDS), + ("Fabric SQL endpoint", FABRIC_ENDPOINT_RE, lambda m: True), + ("personal filesystem path", PERSONAL_PATH_RE, lambda m: True), +] + + +def tracked_files() -> list[str]: + out = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + return [p for p in out.split("\0") if p] + + +def scan_file(rel_path: str) -> list[str]: + path = REPO_ROOT / rel_path + if path.suffix.lower() in BINARY_SUFFIXES or not path.is_file(): + return [] + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return [] + + findings = [] + for lineno, line in enumerate(text.splitlines(), start=1): + for label, pattern, keep in CHECKS: + for match in pattern.finditer(line): + if keep(match): + findings.append( + f"{rel_path}:{lineno}: {label}: {match.group(0)}" + ) + lowered = line.lower() + for literal in FORBIDDEN_LITERALS: + if literal in lowered: + findings.append( + f"{rel_path}:{lineno}: author-specific marker: {literal}" + ) + return findings + + +def main() -> int: + findings: list[str] = [] + for rel_path in tracked_files(): + findings.extend(scan_file(rel_path)) + + if findings: + print("Client / tenant data leak detected in Git-tracked files:\n") + for finding in findings: + print(f" {finding}") + print( + "\nReplace real values with visibly fake placeholders " + "(e.g. or 00000000-0000-0000-0000-000000000000)." + ) + return 1 + + print("No client data leak detected in Git-tracked files.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taskflow/event_taskflow.json b/taskflow/event_taskflow.json index adf86a8..63188e6 100644 --- a/taskflow/event_taskflow.json +++ b/taskflow/event_taskflow.json @@ -2,28 +2,28 @@ "name": "Live Event Operations — Fabric IQ", "description": "End-to-end live-event demo: BIM topology + live telemetry unified in one ontology, consumed by autonomous + interactive agents and 4 persona dashboards.", "tasks": [ - { "type": "get data", "id": "11111111-1111-4111-8111-111111111111", "name": "Ingest topology & telemetry", "description": "Lakehouse (BIM/topology CSVs to Delta) + Eventhouse/KQL (occupancy & queue telemetry)." }, - { "type": "store data", "id": "22222222-2222-4222-8222-222222222222", "name": "Lakehouse — topology", "description": "Edition, Zone (BIM m2), Gate, Sensor, Session, Customer, Sponsorship, Observation." }, - { "type": "store data", "id": "33333333-3333-4333-8333-333333333333", "name": "Eventhouse — live telemetry", "description": "telemetry_kpi (zones), telemetry_queue (gates), alarms, event_logs." }, - { "type": "analyze and train data","id": "44444444-4444-4444-8444-444444444444", "name": "Ontology + Graph (Fabric IQ)", "description": "Unify topology + telemetry; multi-hop GQL for RCA and sponsor impact." }, - { "type": "track data", "id": "55555555-5555-4555-8555-555555555555", "name": "Operations Agent", "description": "Autonomous detection: gate congestion, zone saturation, density, comfort." }, - { "type": "track data", "id": "66666666-6666-4666-8666-666666666666", "name": "Data Activator (Reflex)", "description": "Low-code threshold alerting on telemetry_queue." }, - { "type": "analyze and train data","id": "77777777-7777-4777-8777-777777777777", "name": "Event Graph Data Agent", "description": "NL to GQL/KQL Q&A: session fill, wait-time peaks, VIP sponsor impact." }, - { "type": "visualize", "id": "88888888-8888-4888-8888-888888888888", "name": "RTI Event Dashboard", "description": "Real-time KQL dashboard, 4 persona views: Management, Production, Chefs de projet, Client." }, - { "type": "prepare data", "id": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "name": "Semantic Model (Direct Lake)", "description": "DirectLake model over topology (Lakehouse) + telemetry (Eventhouse); measures for the 4 personas." }, - { "type": "visualize", "id": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "name": "Power BI Report (4 personas)", "description": "RPT_Event_Ops — Direction, Production, Chefs de projet, Client (banner + KPI cards + charts)." } + { "type": "get data", "id": "task-ingest", "name": "Ingest topology & telemetry", "description": "Lakehouse (BIM/topology CSVs to Delta) + Eventhouse/KQL (occupancy & queue telemetry)." }, + { "type": "store data", "id": "task-lakehouse-topology", "name": "Lakehouse — topology", "description": "Edition, Zone (BIM m2), Gate, Sensor, Session, Customer, Sponsorship, Observation." }, + { "type": "store data", "id": "task-eventhouse-telemetry", "name": "Eventhouse — live telemetry", "description": "telemetry_kpi (zones), telemetry_queue (gates), alarms, event_logs." }, + { "type": "analyze and train data","id": "task-ontology-graph", "name": "Ontology + Graph (Fabric IQ)", "description": "Unify topology + telemetry; multi-hop GQL for RCA and sponsor impact." }, + { "type": "track data", "id": "task-operations-agent", "name": "Operations Agent", "description": "Autonomous detection: gate congestion, zone saturation, density, comfort." }, + { "type": "track data", "id": "task-data-activator", "name": "Data Activator (Reflex)", "description": "Low-code threshold alerting on telemetry_queue." }, + { "type": "analyze and train data","id": "task-data-agent", "name": "Event Graph Data Agent", "description": "NL to GQL/KQL Q&A: session fill, wait-time peaks, VIP sponsor impact." }, + { "type": "visualize", "id": "task-rti-dashboard", "name": "RTI Event Dashboard", "description": "Real-time KQL dashboard, 4 persona views: Management, Production, Chefs de projet, Client." }, + { "type": "prepare data", "id": "task-semantic-model", "name": "Semantic Model (Direct Lake)", "description": "DirectLake model over topology (Lakehouse) + telemetry (Eventhouse); measures for the 4 personas." }, + { "type": "visualize", "id": "task-powerbi-report", "name": "Power BI Report (4 personas)", "description": "RPT_Event_Ops — Direction, Production, Chefs de projet, Client (banner + KPI cards + charts)." } ], "edges": [ - { "source": "11111111-1111-4111-8111-111111111111", "target": "22222222-2222-4222-8222-222222222222" }, - { "source": "11111111-1111-4111-8111-111111111111", "target": "33333333-3333-4333-8333-333333333333" }, - { "source": "22222222-2222-4222-8222-222222222222", "target": "44444444-4444-4444-8444-444444444444" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "44444444-4444-4444-8444-444444444444" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "55555555-5555-4555-8555-555555555555" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "66666666-6666-4666-8666-666666666666" }, - { "source": "44444444-4444-4444-8444-444444444444", "target": "77777777-7777-4777-8777-777777777777" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "88888888-8888-4888-8888-888888888888" }, - { "source": "22222222-2222-4222-8222-222222222222", "target": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }, - { "source": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "target": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" } + { "source": "task-ingest", "target": "task-lakehouse-topology" }, + { "source": "task-ingest", "target": "task-eventhouse-telemetry" }, + { "source": "task-lakehouse-topology", "target": "task-ontology-graph" }, + { "source": "task-eventhouse-telemetry", "target": "task-ontology-graph" }, + { "source": "task-eventhouse-telemetry", "target": "task-operations-agent" }, + { "source": "task-eventhouse-telemetry", "target": "task-data-activator" }, + { "source": "task-ontology-graph", "target": "task-data-agent" }, + { "source": "task-eventhouse-telemetry", "target": "task-rti-dashboard" }, + { "source": "task-lakehouse-topology", "target": "task-semantic-model" }, + { "source": "task-eventhouse-telemetry", "target": "task-semantic-model" }, + { "source": "task-semantic-model", "target": "task-powerbi-report" } ] } diff --git a/tests/test_client_denylist.py b/tests/test_client_denylist.py new file mode 100644 index 0000000..4f6f756 --- /dev/null +++ b/tests/test_client_denylist.py @@ -0,0 +1,177 @@ +"""Tests for the client deny-list scanner (scripts/check_client_denylist.py). + +The scanner replaces a hardcoded list of customer names that used to live in the +public CI workflow. These tests pin the three properties that make the +replacement safe: + +* an absent deny-list degrades to a warning, never a failure; +* a deny-listed name in a tracked file fails the build; +* the failure output never echoes the name it matched. + +Every case runs against a throwaway git repo: the scanner only reads +``git ls-files``, so there has to be a real index for it to read. +""" +import pathlib +import shutil +import subprocess +import sys + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SCANNER = ROOT / "scripts" / "check_client_denylist.py" + +# A word that is not a real customer name — the point of this whole exercise is +# that no real name appears in a public repo, tests included. +FAKE_NAME = "zorblatt" + + +def _git(repo: pathlib.Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True) + + +@pytest.fixture +def repo(tmp_path): + """A minimal git repo carrying a copy of the scanner.""" + _git(tmp_path, "init", "-q") + (tmp_path / "scripts").mkdir() + shutil.copy2(SCANNER, tmp_path / "scripts" / SCANNER.name) + return tmp_path + + +def _run(repo: pathlib.Path, denylist: str | None = None): + import os + + env = dict(os.environ) + env.pop("CLIENT_DENYLIST", None) + if denylist is not None: + env["CLIENT_DENYLIST"] = denylist + return subprocess.run( + [sys.executable, str(repo / "scripts" / SCANNER.name)], + cwd=repo, + capture_output=True, + text=True, + env=env, + ) + + +def _track(repo: pathlib.Path, name: str, content: str) -> None: + (repo / name).write_text(content, encoding="utf-8") + _git(repo, "add", name) + + +def test_missing_denylist_warns_but_passes(repo): + """No secret configured (forks, external PRs) must never break the build.""" + _track(repo, "notes.md", f"the {FAKE_NAME} account\n") + result = _run(repo, denylist=None) + assert result.returncode == 0, result.stdout + result.stderr + assert "::warning::" in result.stdout + + +def test_empty_denylist_is_treated_as_absent(repo): + _track(repo, "notes.md", f"the {FAKE_NAME} account\n") + result = _run(repo, denylist="\n \n# only a comment\n") + assert result.returncode == 0 + assert "::warning::" in result.stdout + + +def test_denylisted_name_fails(repo): + _track(repo, "notes.md", f"deployed for {FAKE_NAME} last quarter\n") + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 1, result.stdout + assert "notes.md:1" in result.stdout + + +def test_failure_output_never_echoes_the_name(repo): + """CI logs of a public repo are public — the scanner must stay silent.""" + _track(repo, "notes.md", f"deployed for {FAKE_NAME} last quarter\n") + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 1 + combined = (result.stdout + result.stderr).lower() + assert FAKE_NAME not in combined + assert "last quarter" not in combined + + +def test_separator_variants_are_caught(repo): + """Splitting a name with a dash or a dot is the cheapest obfuscation.""" + _track(repo, "notes.md", "contact: zorb-latt and zorb.latt and zorblatt\n") + result = _run(repo, denylist="zorb latt") + assert result.returncode == 1 + + +def test_parenthesised_letter_obfuscation_is_caught(repo): + """The exact trick this change removes: `z(o)rblatt` reads fine, greps as nothing. + + A regex with a capturing group still matches the plain word, so the guard + kept working while the name hid from full-text search. Stripping single-char + parentheses before matching makes the dodge worthless. + """ + _track(repo, "ci.yml", "PATTERN='z(o)rblatt|something-else'\n") + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 1, result.stdout + + +def test_colour_context_is_ignored(repo): + """Some client names are also colour words; badges and CSS are not leaks.""" + _track( + repo, + "README.md", + "![Build](https://img.shields.io/badge/build-passing-zorblatt)\n", + ) + _track(repo, "theme.css", ".hero { background-color: zorblatt; }\n") + _track(repo, "palette.json", '{ "color": "zorblatt" }\n') + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 0, result.stdout + # Green because the hits were dropped as colour, not because the list was + # never loaded -- a test that passes vacuously protects nothing. + assert "1 entries checked" in result.stdout + + +def test_untracked_files_are_not_scanned(repo): + """Only git ls-files is walked: __pycache__ / node_modules would be noise.""" + (repo / "scratch.md").write_text(f"{FAKE_NAME}\n", encoding="utf-8") + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 0, result.stdout + assert "1 entries checked" in result.stdout + + +def test_local_clientdeny_file_is_used_when_env_is_absent(repo): + _track(repo, "notes.md", f"{FAKE_NAME} again\n") + (repo / ".clientdeny").write_text(f"# local list\n{FAKE_NAME}\n", encoding="utf-8") + result = _run(repo, denylist=None) + assert result.returncode == 1, result.stdout + assert FAKE_NAME not in (result.stdout + result.stderr).lower() + + +def test_canonical_scanner_is_byte_identical(): + """scripts/check_no_client_leak.py is shared verbatim with the sister repos. + + Editing it locally is how two repositories start converging on each other + instead of on the canonical source, so the hash is pinned here. + """ + import hashlib + + canonical = ROOT / "scripts" / "check_no_client_leak.py" + data = canonical.read_bytes() + assert len(data) == 4023, f"expected 4023 bytes, got {len(data)}" + assert ( + hashlib.sha256(data).hexdigest().upper() + == "33312E7E678C09663A7DA51151359F8244EA0CADD10E811FC2A31CCF38328CD3" + ), "check_no_client_leak.py drifted from the canonical copy — re-fetch it, do not edit it" + + +def test_workflow_carries_no_client_names(): + """The guard file must not itself be the leak (this is the bug being fixed). + + A name written with parenthesised letters still matches as an ERE while + hiding from full-text search, so the shape is banned outright. + """ + import re + + workflow = (ROOT / ".github" / "workflows" / "no-client-leak.yml").read_text( + encoding="utf-8" + ) + assert "PATTERN=" not in workflow, "an inline name pattern is back in the workflow" + # A single parenthesised letter inside a word: the obfuscation that was removed. + assert not re.search(r"[A-Za-z]\([A-Za-z]\)[A-Za-z]", workflow), \ + "obfuscated literal detected in the workflow — names belong in a secret"