From 290a41fe06bcfc69cc37c76fca79f85918afb807 Mon Sep 17 00:00:00 2001 From: Statyx Date: Mon, 3 Aug 2026 16:27:30 +0200 Subject: [PATCH] ci: stop publishing the client list in the leak guard The workflow carried the full customer roster inline, each name written with one letter in parentheses. An ERE capturing group matches the same string, so the scan kept working while a full-text search for the name found nothing -- that fools GitHub's index, not a reader. The guard against leaking a client name had become the leak, and in a worse shape than a stray mention: the whole portfolio, concentrated in one file of a public repo. Detection is now by SHAPE, never by name: * scripts/check_no_client_leak.py -- copied byte-for-byte from the canonical source (Statyx/Fab-Analyze-Data-Agent@6e4a65e7, sha256 33312e7e...28cd3). Real GUIDs, real Fabric SQL endpoints and personal paths, over `git ls-files` only. Not to be edited here: a local improvement is how two repos start converging on each other instead of on the canonical copy. * scripts/check_client_denylist.py -- name matching is preserved, but the names come from the CLIENT_DENYLIST secret (or a gitignored .clientdeny). No secret: warning, build stays green. Findings print a file, a line and a hash prefix -- never the name, never the line, since public CI logs would republish it. Single-character parentheses are stripped before matching, so the obfuscation removed here cannot be reintroduced unnoticed. Colour contexts (shields.io, hex, rgb(), CSS declarations) are dropped generically, because some client names double as colour words. The canonical checker only allowlists the nil GUID, so the visibly fake repeated-digit placeholders in the task flow tripped it. The ids there are internal to that file, so they become readable slugs; edge integrity was verified (10 tasks, 11 edges, 0 orphans). .gitattributes pins LF on both scripts: core.autocrlf=true would rewrite the canonical copy on checkout and break its hash on Windows. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .gitattributes | 6 + .github/workflows/no-client-leak.yml | 46 +++---- .gitignore | 5 + README.md | 5 +- scripts/check_client_denylist.py | 178 +++++++++++++++++++++++++++ scripts/check_no_client_leak.py | 130 +++++++++++++++++++ taskflow/event_taskflow.json | 42 +++---- tests/test_client_denylist.py | 177 ++++++++++++++++++++++++++ 8 files changed, 537 insertions(+), 52 deletions(-) create mode 100644 .gitattributes create mode 100644 scripts/check_client_denylist.py create mode 100644 scripts/check_no_client_leak.py create mode 100644 tests/test_client_denylist.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1be5786 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# scripts/check_no_client_leak.py is copied byte-for-byte from the canonical source +# (Statyx/Fab-Analyze-Data-Agent). Its SHA-256 is the proof it was not edited, so it must +# keep LF endings on every platform — core.autocrlf=true would otherwise rewrite it to CRLF +# on checkout and break the hash for anyone verifying it on Windows. +scripts/check_no_client_leak.py text eol=lf +scripts/check_client_denylist.py text eol=lf diff --git a/.github/workflows/no-client-leak.yml b/.github/workflows/no-client-leak.yml index 8f3d023..a549944 100644 --- a/.github/workflows/no-client-leak.yml +++ b/.github/workflows/no-client-leak.yml @@ -13,36 +13,24 @@ jobs: steps: - uses: actions/checkout@v4 - # This repo is public. Any real customer name or hardcoded Entra/Fabric identifier - # is a leak, so the build fails loudly instead of quietly shipping it. - # The names below are written with a single letter parenthesised (ERE groups match the - # same string) so that this guard file is not itself a hit for its own scan. - - name: Scan for client names - run: | - PATTERN='p(u)blicis|v(i)vatech|o(r)ange|a(i)r.?liquide|t(e)chnip|f(o)rvia|v(e)olia|a(r)kema|c(a)pgemini|e(i)ffage|o(p)mobility' - if grep -rniE "$PATTERN" \ - --exclude-dir=.git \ - --exclude-dir=node_modules \ - --exclude-dir=__pycache__ \ - . ; then - echo "::error::A known client name appears in the repository. Anonymise it before merging." - exit 1 - fi - echo "OK - no client name found." + - uses: actions/setup-python@v5 + with: + python-version: "3.12" - - name: Scan for hardcoded clientId / tenantId GUIDs - run: | - PATTERN='(client_?id|tenant_?id)["'"'"']?[[:space:]]*[:=][[:space:]]*["'"'"'][0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}' - if grep -rniE "$PATTERN" \ - --exclude-dir=.git \ - --exclude-dir=node_modules \ - --exclude-dir=__pycache__ \ - --exclude-dir=workflows \ - . ; then - echo "::error::A clientId/tenantId GUID is hardcoded. Inject it from the environment instead." - exit 1 - fi - echo "OK - no hardcoded clientId/tenantId." + # This repo is public. Detection is by SHAPE, never by name: a list of customer + # names committed here would itself be the leak it is meant to prevent, and + # obfuscating that list only fools full-text search, not a reader. + # scripts/check_no_client_leak.py is copied byte-for-byte from the canonical + # source (Statyx/Fab-Analyze-Data-Agent@6e4a65e7) — do not edit it locally. + - name: Structural scan (GUIDs, Fabric endpoints, personal paths) + run: python scripts/check_no_client_leak.py + + # Name-based detection is preserved, but the names live in a secret, never in + # the source tree. Secret missing (forks, external PRs) -> warning, still green. + - name: Client deny-list scan (from secret, optional) + env: + CLIENT_DENYLIST: ${{ secrets.CLIENT_DENYLIST }} + run: python scripts/check_client_denylist.py tests: name: pytest diff --git a/.gitignore b/.gitignore index 998b4ff..a1ed0ac 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,11 @@ _*.json Thumbs.db *.swp +# Local client deny-list read by scripts/check_client_denylist.py. It holds real customer +# names, so it must never be committed to this public repo — in CI the same list comes from +# the CLIENT_DENYLIST secret instead. +.clientdeny + # Environment-specific — contain tenant / workspace / item IDs. Use the *.example files as templates. src/config.yaml src/state.json diff --git a/README.md b/README.md index 9c0d8a2..44d881f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ![Microsoft Fabric](https://img.shields.io/badge/Microsoft-Fabric-0078D4?logo=microsoft&logoColor=white) ![Fabric IQ](https://img.shields.io/badge/Fabric_IQ-Ontology_+_Graph-6242C9) ![Python](https://img.shields.io/badge/Python-3.12-3776AB?logo=python&logoColor=white) -![Tests](https://img.shields.io/badge/tests-39_passing-2E9E44) +![Tests](https://img.shields.io/badge/tests-50_passing-2E9E44) ![Portal](https://img.shields.io/badge/Portal-FastAPI-009688?logo=fastapi&logoColor=white) ![Deploy](https://img.shields.io/badge/deploy-idempotent-896610) @@ -77,7 +77,7 @@ flowchart LR copy src\config.example.yaml src\config.yaml # 2. ✅ Mandatory test gate — never deploy on red -python -m pytest tests/ -v --tb=short # 39/39 offline +python -m pytest tests/ -v --tb=short # 50/50 offline # 3. Deploy everything (idempotent — generates data, then ends with a warm-up) python src\deploy_all.py @@ -194,6 +194,7 @@ node build_deck.js # → presentation/Architecture_Vision.pptx - ✅ **Test gate is mandatory** — `python -m pytest tests/ -v` before any `deploy_*.py`. On red: **stop, fix the code**. - ♻️ **Idempotent deploys** — every step reads/writes `src/state.json`; re-running is safe. - 🔐 **Secrets never committed** — `src/config.yaml`, `src/state.json`, `.vscode/mcp.json` are gitignored (use the `*.example` templates). No tenant / workspace / app-registration ID is hardcoded anywhere; CI enforces this. +- 🕵️ **Leak detection is by shape, not by name** — `scripts/check_no_client_leak.py` (copied byte-for-byte from the canonical source, do not edit it) flags real GUIDs, real Fabric SQL endpoints and personal paths in Git-tracked files. Customer names are matched separately by `scripts/check_client_denylist.py`, whose list comes from the `CLIENT_DENYLIST` repository secret (one name per line) or a gitignored local `.clientdeny`. **The names are never committed to this public repo** — that list would be the leak. No secret configured: the rule warns and passes. - 🧪 **Synthetic data only** — every organisation, venue and sponsor in this repo is fictional. - ⚡ **F2+ capacity** — resume the capacity before a deploy/demo (Ontology / Graph / agents need it). - 🧠 **Reports are Legacy PBIX** — see the report gotchas below. diff --git a/scripts/check_client_denylist.py b/scripts/check_client_denylist.py new file mode 100644 index 0000000..686908c --- /dev/null +++ b/scripts/check_client_denylist.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Fail if a name from the private client deny-list appears in Git-tracked files. + +This repo is public, so the deny-list itself must never be committed here. The +list is injected at run time and stays out of the source tree: + + 1. ``CLIENT_DENYLIST`` -- one entry per line (GitHub Actions secret). + 2. ``.clientdeny`` at the repo root -- gitignored, for local runs. + +Neither present? The rule is skipped with a warning and the build stays green. +An absent secret must never break a fork, a PR from outside the org, or a +contributor's laptop. + +Only files returned by ``git ls-files`` are scanned, for the same reason as the +structural checker: ``__pycache__/*.pyc`` and ``node_modules`` embed absolute +paths and produce false positives. + +Two deliberate properties: + +* **Nothing is echoed.** A finding prints the file, the line number and a hash + prefix of the entry -- never the entry itself, never the source line. CI logs + of a public repo are public: a chatty scanner would re-publish the very name + it is guarding. +* **Colour contexts are ignored.** Some client names double as ordinary colour + words. A hit on a line that is plainly about colour -- a shields.io badge, a + hex token, ``rgb()``, a CSS declaration -- is dropped. The rule is generic; no + name and no colour is hardcoded here. + +Usage: + python scripts/check_client_denylist.py + +Exit code 0 = clean or not configured, 1 = a deny-listed name was found. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +LOCAL_DENYLIST = REPO_ROOT / ".clientdeny" + +BINARY_SUFFIXES = { + ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".pptx", ".xlsx", + ".docx", ".zip", ".gz", ".woff", ".woff2", ".ttf", ".eot", +} + +# A line that is plainly about colour. Generic on purpose: it keys on the +# colour syntax around the hit, not on which word was matched. +COLOUR_CONTEXT_RE = re.compile( + r""" + img\.shields\.io # badge URLs: .../badge/x-y- + | \#[0-9A-Fa-f]{3,8}\b # hex token + | \brgba?\s*\( # rgb() / rgba() + | \bhsla?\s*\( # hsl() / hsla() + | (?:^|[;{,\s])(?:color|background|background-color|fill|stroke|border + |border-color|outline|box-shadow|--[\w-]+)\s*: # CSS declaration + | "(?:color|colour|fill|stroke|background)"\s*: # JSON theme key + """, + re.IGNORECASE | re.VERBOSE, +) + +# The obfuscation this whole change exists to remove: a single character wrapped +# in parentheses. `n(a)me` still matches as an ERE and reads fine to a human, but +# no full-text search for the name finds it. Parentheses are stripped before +# matching so the trick buys nothing. +SINGLE_CHAR_PAREN_RE = re.compile(r"\(([A-Za-z0-9])\)") + + +def load_denylist() -> tuple[list[str], str]: + """Return (entries, source-label). Empty list means "not configured".""" + raw = os.environ.get("CLIENT_DENYLIST", "") + source = "the CLIENT_DENYLIST environment variable" + if not raw.strip() and LOCAL_DENYLIST.is_file(): + raw = LOCAL_DENYLIST.read_text(encoding="utf-8", errors="ignore") + source = str(LOCAL_DENYLIST.name) + + entries = [] + for line in raw.splitlines(): + entry = line.strip() + if entry and not entry.startswith("#"): + entries.append(entry) + return entries, source + + +def compile_entry(entry: str) -> re.Pattern[str]: + """Word-bounded, case-insensitive, separator-tolerant literal match. + + A two-word entry also catches the hyphenated, underscored and glued + spellings -- the separator is where obfuscation is cheapest. + """ + parts = [re.escape(part) for part in entry.split()] + body = r"[\s._\-]*".join(parts) + return re.compile(rf"\b{body}\b", re.IGNORECASE) + + +def fingerprint(entry: str) -> str: + """Stable, non-reversible label so a maintainer can tell entries apart.""" + digest = hashlib.sha256(entry.strip().lower().encode("utf-8")).hexdigest() + return digest[:8] + + +def tracked_files() -> list[str]: + out = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + return [p for p in out.split("\0") if p] + + +def scan_file(rel_path: str, patterns: list[tuple[int, str, re.Pattern[str]]]) -> list[str]: + path = REPO_ROOT / rel_path + if path.suffix.lower() in BINARY_SUFFIXES or not path.is_file(): + return [] + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return [] + + findings = [] + for lineno, line in enumerate(text.splitlines(), start=1): + colour_line = COLOUR_CONTEXT_RE.search(line) is not None + if colour_line: + continue # the word is being used as a colour, not as a name + variants = {line, SINGLE_CHAR_PAREN_RE.sub(r"\1", line)} + for index, mark, pattern in patterns: + if any(pattern.search(variant) for variant in variants): + findings.append(f"{rel_path}:{lineno}: deny-list entry #{index} [{mark}]") + return findings + + +def main() -> int: + entries, source = load_denylist() + if not entries: + print( + "::warning::Client deny-list not configured " + "(set the CLIENT_DENYLIST secret, or a gitignored .clientdeny file) " + "- name-based scanning skipped. Structural checks still ran." + ) + return 0 + + patterns = [ + (index, fingerprint(entry), compile_entry(entry)) + for index, entry in enumerate(entries, start=1) + ] + + findings: list[str] = [] + for rel_path in tracked_files(): + findings.extend(scan_file(rel_path, patterns)) + + if findings: + # Entries are reported by index + hash prefix only: printing the name + # here would leak it into public CI logs. + print( + f"Deny-listed client name(s) found in Git-tracked files " + f"({len(entries)} entries loaded from {source}):\n" + ) + for finding in findings: + print(f" {finding}") + print( + "\nThe matched text is deliberately not shown. Open the file at the " + "line above and anonymise it before merging." + ) + return 1 + + print(f"No deny-listed client name found ({len(entries)} entries checked).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_no_client_leak.py b/scripts/check_no_client_leak.py new file mode 100644 index 0000000..bf1b825 --- /dev/null +++ b/scripts/check_no_client_leak.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Fail if real customer / tenant identifiers leak into Git-tracked files. + +This repo is public. It must never contain real Microsoft Fabric identifiers, +real workspace SQL endpoints, or personal filesystem paths. + +Only files returned by ``git ls-files`` are scanned. The working tree is +deliberately NOT walked: ``__pycache__/*.pyc`` and ``node_modules`` embed +absolute build paths and would produce false positives. + +Usage: + python scripts/check_no_client_leak.py + +Exit code 0 = clean, 1 = leak detected. +""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# Placeholder GUIDs that are allowed to appear verbatim. +ALLOWED_GUIDS = { + "00000000-0000-0000-0000-000000000000", +} + +BINARY_SUFFIXES = { + ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".pptx", ".xlsx", + ".docx", ".zip", ".gz", ".woff", ".woff2", ".ttf", ".eot", +} + +GUID_RE = re.compile( + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-" + r"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b" +) + +# A real Fabric SQL/warehouse endpoint host: a long opaque token followed by +# ".datawarehouse.fabric.microsoft.com" (or the Synapse / Power BI equivalents). +FABRIC_ENDPOINT_RE = re.compile( + r"\b[a-z0-9]{20,}(?:-[a-z0-9]{20,})?\." + r"(?:datawarehouse|dev\.azuresynapse|pbidedicated)\b", + re.IGNORECASE, +) + +# Personal home directories. Placeholders such as C:\Users\ are fine. +PERSONAL_PATH_RE = re.compile( + r"(?:[A-Za-z]:\\Users\\|/Users/|/home/)(?!<|\{|\$|%|USERNAME\b|username\b|you\b|user\b)" + r"[A-Za-z0-9._-]+", +) + +# Extra literals to forbid, as a comma-separated list. Deliberately empty by +# default: hardcoding an author's username here would re-publish the very thing +# this checker exists to remove. +# NO_LEAK_EXTRA_LITERALS="acme,contoso" python scripts/check_no_client_leak.py +FORBIDDEN_LITERALS = [ + literal.strip().lower() + for literal in os.environ.get("NO_LEAK_EXTRA_LITERALS", "").split(",") + if literal.strip() +] + +CHECKS = [ + ("real GUID", GUID_RE, lambda m: m.group(0).lower() not in ALLOWED_GUIDS), + ("Fabric SQL endpoint", FABRIC_ENDPOINT_RE, lambda m: True), + ("personal filesystem path", PERSONAL_PATH_RE, lambda m: True), +] + + +def tracked_files() -> list[str]: + out = subprocess.run( + ["git", "ls-files", "-z"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ).stdout + return [p for p in out.split("\0") if p] + + +def scan_file(rel_path: str) -> list[str]: + path = REPO_ROOT / rel_path + if path.suffix.lower() in BINARY_SUFFIXES or not path.is_file(): + return [] + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return [] + + findings = [] + for lineno, line in enumerate(text.splitlines(), start=1): + for label, pattern, keep in CHECKS: + for match in pattern.finditer(line): + if keep(match): + findings.append( + f"{rel_path}:{lineno}: {label}: {match.group(0)}" + ) + lowered = line.lower() + for literal in FORBIDDEN_LITERALS: + if literal in lowered: + findings.append( + f"{rel_path}:{lineno}: author-specific marker: {literal}" + ) + return findings + + +def main() -> int: + findings: list[str] = [] + for rel_path in tracked_files(): + findings.extend(scan_file(rel_path)) + + if findings: + print("Client / tenant data leak detected in Git-tracked files:\n") + for finding in findings: + print(f" {finding}") + print( + "\nReplace real values with visibly fake placeholders " + "(e.g. or 00000000-0000-0000-0000-000000000000)." + ) + return 1 + + print("No client data leak detected in Git-tracked files.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/taskflow/event_taskflow.json b/taskflow/event_taskflow.json index adf86a8..63188e6 100644 --- a/taskflow/event_taskflow.json +++ b/taskflow/event_taskflow.json @@ -2,28 +2,28 @@ "name": "Live Event Operations — Fabric IQ", "description": "End-to-end live-event demo: BIM topology + live telemetry unified in one ontology, consumed by autonomous + interactive agents and 4 persona dashboards.", "tasks": [ - { "type": "get data", "id": "11111111-1111-4111-8111-111111111111", "name": "Ingest topology & telemetry", "description": "Lakehouse (BIM/topology CSVs to Delta) + Eventhouse/KQL (occupancy & queue telemetry)." }, - { "type": "store data", "id": "22222222-2222-4222-8222-222222222222", "name": "Lakehouse — topology", "description": "Edition, Zone (BIM m2), Gate, Sensor, Session, Customer, Sponsorship, Observation." }, - { "type": "store data", "id": "33333333-3333-4333-8333-333333333333", "name": "Eventhouse — live telemetry", "description": "telemetry_kpi (zones), telemetry_queue (gates), alarms, event_logs." }, - { "type": "analyze and train data","id": "44444444-4444-4444-8444-444444444444", "name": "Ontology + Graph (Fabric IQ)", "description": "Unify topology + telemetry; multi-hop GQL for RCA and sponsor impact." }, - { "type": "track data", "id": "55555555-5555-4555-8555-555555555555", "name": "Operations Agent", "description": "Autonomous detection: gate congestion, zone saturation, density, comfort." }, - { "type": "track data", "id": "66666666-6666-4666-8666-666666666666", "name": "Data Activator (Reflex)", "description": "Low-code threshold alerting on telemetry_queue." }, - { "type": "analyze and train data","id": "77777777-7777-4777-8777-777777777777", "name": "Event Graph Data Agent", "description": "NL to GQL/KQL Q&A: session fill, wait-time peaks, VIP sponsor impact." }, - { "type": "visualize", "id": "88888888-8888-4888-8888-888888888888", "name": "RTI Event Dashboard", "description": "Real-time KQL dashboard, 4 persona views: Management, Production, Chefs de projet, Client." }, - { "type": "prepare data", "id": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "name": "Semantic Model (Direct Lake)", "description": "DirectLake model over topology (Lakehouse) + telemetry (Eventhouse); measures for the 4 personas." }, - { "type": "visualize", "id": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", "name": "Power BI Report (4 personas)", "description": "RPT_Event_Ops — Direction, Production, Chefs de projet, Client (banner + KPI cards + charts)." } + { "type": "get data", "id": "task-ingest", "name": "Ingest topology & telemetry", "description": "Lakehouse (BIM/topology CSVs to Delta) + Eventhouse/KQL (occupancy & queue telemetry)." }, + { "type": "store data", "id": "task-lakehouse-topology", "name": "Lakehouse — topology", "description": "Edition, Zone (BIM m2), Gate, Sensor, Session, Customer, Sponsorship, Observation." }, + { "type": "store data", "id": "task-eventhouse-telemetry", "name": "Eventhouse — live telemetry", "description": "telemetry_kpi (zones), telemetry_queue (gates), alarms, event_logs." }, + { "type": "analyze and train data","id": "task-ontology-graph", "name": "Ontology + Graph (Fabric IQ)", "description": "Unify topology + telemetry; multi-hop GQL for RCA and sponsor impact." }, + { "type": "track data", "id": "task-operations-agent", "name": "Operations Agent", "description": "Autonomous detection: gate congestion, zone saturation, density, comfort." }, + { "type": "track data", "id": "task-data-activator", "name": "Data Activator (Reflex)", "description": "Low-code threshold alerting on telemetry_queue." }, + { "type": "analyze and train data","id": "task-data-agent", "name": "Event Graph Data Agent", "description": "NL to GQL/KQL Q&A: session fill, wait-time peaks, VIP sponsor impact." }, + { "type": "visualize", "id": "task-rti-dashboard", "name": "RTI Event Dashboard", "description": "Real-time KQL dashboard, 4 persona views: Management, Production, Chefs de projet, Client." }, + { "type": "prepare data", "id": "task-semantic-model", "name": "Semantic Model (Direct Lake)", "description": "DirectLake model over topology (Lakehouse) + telemetry (Eventhouse); measures for the 4 personas." }, + { "type": "visualize", "id": "task-powerbi-report", "name": "Power BI Report (4 personas)", "description": "RPT_Event_Ops — Direction, Production, Chefs de projet, Client (banner + KPI cards + charts)." } ], "edges": [ - { "source": "11111111-1111-4111-8111-111111111111", "target": "22222222-2222-4222-8222-222222222222" }, - { "source": "11111111-1111-4111-8111-111111111111", "target": "33333333-3333-4333-8333-333333333333" }, - { "source": "22222222-2222-4222-8222-222222222222", "target": "44444444-4444-4444-8444-444444444444" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "44444444-4444-4444-8444-444444444444" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "55555555-5555-4555-8555-555555555555" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "66666666-6666-4666-8666-666666666666" }, - { "source": "44444444-4444-4444-8444-444444444444", "target": "77777777-7777-4777-8777-777777777777" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "88888888-8888-4888-8888-888888888888" }, - { "source": "22222222-2222-4222-8222-222222222222", "target": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }, - { "source": "33333333-3333-4333-8333-333333333333", "target": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" }, - { "source": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", "target": "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" } + { "source": "task-ingest", "target": "task-lakehouse-topology" }, + { "source": "task-ingest", "target": "task-eventhouse-telemetry" }, + { "source": "task-lakehouse-topology", "target": "task-ontology-graph" }, + { "source": "task-eventhouse-telemetry", "target": "task-ontology-graph" }, + { "source": "task-eventhouse-telemetry", "target": "task-operations-agent" }, + { "source": "task-eventhouse-telemetry", "target": "task-data-activator" }, + { "source": "task-ontology-graph", "target": "task-data-agent" }, + { "source": "task-eventhouse-telemetry", "target": "task-rti-dashboard" }, + { "source": "task-lakehouse-topology", "target": "task-semantic-model" }, + { "source": "task-eventhouse-telemetry", "target": "task-semantic-model" }, + { "source": "task-semantic-model", "target": "task-powerbi-report" } ] } diff --git a/tests/test_client_denylist.py b/tests/test_client_denylist.py new file mode 100644 index 0000000..4f6f756 --- /dev/null +++ b/tests/test_client_denylist.py @@ -0,0 +1,177 @@ +"""Tests for the client deny-list scanner (scripts/check_client_denylist.py). + +The scanner replaces a hardcoded list of customer names that used to live in the +public CI workflow. These tests pin the three properties that make the +replacement safe: + +* an absent deny-list degrades to a warning, never a failure; +* a deny-listed name in a tracked file fails the build; +* the failure output never echoes the name it matched. + +Every case runs against a throwaway git repo: the scanner only reads +``git ls-files``, so there has to be a real index for it to read. +""" +import pathlib +import shutil +import subprocess +import sys + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SCANNER = ROOT / "scripts" / "check_client_denylist.py" + +# A word that is not a real customer name — the point of this whole exercise is +# that no real name appears in a public repo, tests included. +FAKE_NAME = "zorblatt" + + +def _git(repo: pathlib.Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True) + + +@pytest.fixture +def repo(tmp_path): + """A minimal git repo carrying a copy of the scanner.""" + _git(tmp_path, "init", "-q") + (tmp_path / "scripts").mkdir() + shutil.copy2(SCANNER, tmp_path / "scripts" / SCANNER.name) + return tmp_path + + +def _run(repo: pathlib.Path, denylist: str | None = None): + import os + + env = dict(os.environ) + env.pop("CLIENT_DENYLIST", None) + if denylist is not None: + env["CLIENT_DENYLIST"] = denylist + return subprocess.run( + [sys.executable, str(repo / "scripts" / SCANNER.name)], + cwd=repo, + capture_output=True, + text=True, + env=env, + ) + + +def _track(repo: pathlib.Path, name: str, content: str) -> None: + (repo / name).write_text(content, encoding="utf-8") + _git(repo, "add", name) + + +def test_missing_denylist_warns_but_passes(repo): + """No secret configured (forks, external PRs) must never break the build.""" + _track(repo, "notes.md", f"the {FAKE_NAME} account\n") + result = _run(repo, denylist=None) + assert result.returncode == 0, result.stdout + result.stderr + assert "::warning::" in result.stdout + + +def test_empty_denylist_is_treated_as_absent(repo): + _track(repo, "notes.md", f"the {FAKE_NAME} account\n") + result = _run(repo, denylist="\n \n# only a comment\n") + assert result.returncode == 0 + assert "::warning::" in result.stdout + + +def test_denylisted_name_fails(repo): + _track(repo, "notes.md", f"deployed for {FAKE_NAME} last quarter\n") + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 1, result.stdout + assert "notes.md:1" in result.stdout + + +def test_failure_output_never_echoes_the_name(repo): + """CI logs of a public repo are public — the scanner must stay silent.""" + _track(repo, "notes.md", f"deployed for {FAKE_NAME} last quarter\n") + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 1 + combined = (result.stdout + result.stderr).lower() + assert FAKE_NAME not in combined + assert "last quarter" not in combined + + +def test_separator_variants_are_caught(repo): + """Splitting a name with a dash or a dot is the cheapest obfuscation.""" + _track(repo, "notes.md", "contact: zorb-latt and zorb.latt and zorblatt\n") + result = _run(repo, denylist="zorb latt") + assert result.returncode == 1 + + +def test_parenthesised_letter_obfuscation_is_caught(repo): + """The exact trick this change removes: `z(o)rblatt` reads fine, greps as nothing. + + A regex with a capturing group still matches the plain word, so the guard + kept working while the name hid from full-text search. Stripping single-char + parentheses before matching makes the dodge worthless. + """ + _track(repo, "ci.yml", "PATTERN='z(o)rblatt|something-else'\n") + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 1, result.stdout + + +def test_colour_context_is_ignored(repo): + """Some client names are also colour words; badges and CSS are not leaks.""" + _track( + repo, + "README.md", + "![Build](https://img.shields.io/badge/build-passing-zorblatt)\n", + ) + _track(repo, "theme.css", ".hero { background-color: zorblatt; }\n") + _track(repo, "palette.json", '{ "color": "zorblatt" }\n') + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 0, result.stdout + # Green because the hits were dropped as colour, not because the list was + # never loaded -- a test that passes vacuously protects nothing. + assert "1 entries checked" in result.stdout + + +def test_untracked_files_are_not_scanned(repo): + """Only git ls-files is walked: __pycache__ / node_modules would be noise.""" + (repo / "scratch.md").write_text(f"{FAKE_NAME}\n", encoding="utf-8") + result = _run(repo, denylist=FAKE_NAME) + assert result.returncode == 0, result.stdout + assert "1 entries checked" in result.stdout + + +def test_local_clientdeny_file_is_used_when_env_is_absent(repo): + _track(repo, "notes.md", f"{FAKE_NAME} again\n") + (repo / ".clientdeny").write_text(f"# local list\n{FAKE_NAME}\n", encoding="utf-8") + result = _run(repo, denylist=None) + assert result.returncode == 1, result.stdout + assert FAKE_NAME not in (result.stdout + result.stderr).lower() + + +def test_canonical_scanner_is_byte_identical(): + """scripts/check_no_client_leak.py is shared verbatim with the sister repos. + + Editing it locally is how two repositories start converging on each other + instead of on the canonical source, so the hash is pinned here. + """ + import hashlib + + canonical = ROOT / "scripts" / "check_no_client_leak.py" + data = canonical.read_bytes() + assert len(data) == 4023, f"expected 4023 bytes, got {len(data)}" + assert ( + hashlib.sha256(data).hexdigest().upper() + == "33312E7E678C09663A7DA51151359F8244EA0CADD10E811FC2A31CCF38328CD3" + ), "check_no_client_leak.py drifted from the canonical copy — re-fetch it, do not edit it" + + +def test_workflow_carries_no_client_names(): + """The guard file must not itself be the leak (this is the bug being fixed). + + A name written with parenthesised letters still matches as an ERE while + hiding from full-text search, so the shape is banned outright. + """ + import re + + workflow = (ROOT / ".github" / "workflows" / "no-client-leak.yml").read_text( + encoding="utf-8" + ) + assert "PATTERN=" not in workflow, "an inline name pattern is back in the workflow" + # A single parenthesised letter inside a word: the obfuscation that was removed. + assert not re.search(r"[A-Za-z]\([A-Za-z]\)[A-Za-z]", workflow), \ + "obfuscated literal detected in the workflow — names belong in a secret"