From 3967f430aab7a47958754bf825aa6e044640c10a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 28 Jun 2026 09:45:21 +0000 Subject: [PATCH 1/5] ci: upgrade secret scanning workflow INF-162 --- .../workflows/reusable-secrets-scanning.yml | 401 ++++++++++++++++-- .github/workflows/secrets-scanning.yml | 24 +- README.md | 69 ++- 3 files changed, 445 insertions(+), 49 deletions(-) diff --git a/.github/workflows/reusable-secrets-scanning.yml b/.github/workflows/reusable-secrets-scanning.yml index f2eec2c..ac269ff 100644 --- a/.github/workflows/reusable-secrets-scanning.yml +++ b/.github/workflows/reusable-secrets-scanning.yml @@ -1,83 +1,396 @@ -name: Reusable TruffleHog Scan +name: Reusable TN Secret Scan on: workflow_call: inputs: branch: - required: true + description: "Branch or ref to scan. Kept for backward compatibility; defaults to the caller ref." + required: false type: string + default: "" depth: + description: "Deprecated. Use fetch_depth instead." required: false type: number default: 2 + scan_mode: + description: "Scan mode: pr, push, or history." + required: false + type: string + default: pr + fetch_depth: + description: "Checkout fetch depth. Use 0 for full history." + required: false + type: number + default: 0 + fail_threshold: + description: "Lowest severity that fails pr/push scans: critical, high, medium, or low." + required: false + type: string + default: high + config_ref: + description: "Ref in treasurenetprotocol/reusable-workflows that contains the shared Gitleaks config." + required: false + type: string + default: main + config_path: + description: "Path to the shared Gitleaks config inside reusable-workflows." + required: false + type: string + default: security/gitleaks/tn-gitleaks.toml + gitleaks_version: + description: "Gitleaks container version." + required: false + type: string + default: v8.27.2 + trufflehog_version: + description: "TruffleHog container version." + required: false + type: string + default: 3.90.5 secrets: SLACK_BOT_TOKEN: required: false SLACK_CHANNEL_ID_GITHUB_NOTIFICATION: required: false -# Permission can be added at job level or workflow level permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout + contents: read jobs: SecurityScan: runs-on: ubuntu-latest env: - BRANCH: ${{ inputs.branch }} - DEPTH: ${{ inputs.depth }} + SCAN_MODE: ${{ inputs.scan_mode }} + FAIL_THRESHOLD: ${{ inputs.fail_threshold }} + GITLEAKS_VERSION: ${{ inputs.gitleaks_version }} + TRUFFLEHOG_VERSION: ${{ inputs.trufflehog_version }} + GITLEAKS_CONFIG: _tn-security-config/${{ inputs.config_path }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} + PUSH_AFTER_SHA: ${{ github.sha }} steps: + - name: Validate inputs + run: | + set -euo pipefail - - name: Checkout the repository + case "${SCAN_MODE}" in + pr|push|history) ;; + *) + echo "::error::scan_mode must be one of: pr, push, history" + exit 2 + ;; + esac + + case "${FAIL_THRESHOLD}" in + critical|high|medium|low) ;; + *) + echo "::error::fail_threshold must be one of: critical, high, medium, low" + exit 2 + ;; + esac + + - name: Checkout caller repository uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #v4.1.4 with: - ref: ${{ env.BRANCH }} - fetch-depth: ${{ env.DEPTH }} + ref: ${{ inputs.branch }} + fetch-depth: ${{ inputs.fetch_depth }} - - name: Run TruffleHog Scan - uses: trufflesecurity/trufflehog@4ea3a1376b709cebb6a5c7b664693fe733bd43f5 #v3.75.0 + - name: Checkout TN scanner configuration + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #v4.1.4 with: - extra_args: --only-verified + repository: treasurenetprotocol/reusable-workflows + ref: ${{ inputs.config_ref }} + path: _tn-security-config + fetch-depth: 1 + + - name: Prepare scan range + id: scan-range + run: | + set -euo pipefail + + mkdir -p .tn-secret-scan + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + gitleaks_log_opts="" + trufflehog_since_commit="" + + case "${SCAN_MODE}" in + pr) + if [ -n "${PR_BASE_SHA}" ] && git cat-file -e "${PR_BASE_SHA}^{commit}" 2>/dev/null; then + gitleaks_log_opts="${PR_BASE_SHA}..${PR_HEAD_SHA:-HEAD}" + trufflehog_since_commit="${PR_BASE_SHA}" + else + echo "::warning::Pull request base SHA is unavailable in this checkout; scanning the checked-out history." + fi + ;; + push) + if [ -n "${PUSH_BEFORE_SHA}" ] && \ + [ "${PUSH_BEFORE_SHA}" != "0000000000000000000000000000000000000000" ] && \ + git cat-file -e "${PUSH_BEFORE_SHA}^{commit}" 2>/dev/null; then + gitleaks_log_opts="${PUSH_BEFORE_SHA}..${PUSH_AFTER_SHA:-HEAD}" + trufflehog_since_commit="${PUSH_BEFORE_SHA}" + else + echo "::warning::Push before SHA is unavailable in this checkout; scanning the checked-out history." + fi + ;; + history) + echo "History mode scans the checked-out repository history and does not block by default." + ;; + esac + + { + echo "gitleaks_log_opts=${gitleaks_log_opts}" + echo "trufflehog_since_commit=${trufflehog_since_commit}" + } >> "$GITHUB_OUTPUT" + + - name: Run Gitleaks + id: gitleaks + run: | + set -euo pipefail + + if [ ! -f "${GITLEAKS_CONFIG}" ]; then + echo "::error::Gitleaks config not found at ${GITLEAKS_CONFIG}" + exit 2 + fi + + args=( + git + --source /repo + --config "/repo/${GITLEAKS_CONFIG}" + --report-format json + --report-path /repo/.tn-secret-scan/gitleaks.json + --redact + --exit-code 0 + ) + + if [ -n "${{ steps.scan-range.outputs.gitleaks_log_opts }}" ]; then + args+=(--log-opts "${{ steps.scan-range.outputs.gitleaks_log_opts }}") + fi + + docker run --rm \ + -v "$GITHUB_WORKSPACE:/repo" \ + "ghcr.io/gitleaks/gitleaks:${GITLEAKS_VERSION}" "${args[@]}" + + - name: Run TruffleHog + id: trufflehog + run: | + set -euo pipefail + + args=( + git + file:///repo + --json + --only-verified + --no-update + ) + + if [ -n "${{ steps.scan-range.outputs.trufflehog_since_commit }}" ]; then + args+=(--since-commit "${{ steps.scan-range.outputs.trufflehog_since_commit }}") + fi + + set +e + docker run --rm \ + -v "$GITHUB_WORKSPACE:/repo" \ + "trufflesecurity/trufflehog:${TRUFFLEHOG_VERSION}" "${args[@]}" \ + > .tn-secret-scan/trufflehog.jsonl + status=$? + set -e + + if [ "$status" -ne 0 ] && [ "$status" -ne 183 ]; then + echo "::error::TruffleHog scan failed before findings could be summarized." + exit "$status" + fi + + - name: Build sanitized secret scan summary + id: summary + run: | + set -euo pipefail + + python3 <<'PY' + import json + import os + import re + import sys + import tomllib + from pathlib import Path + + threshold = os.environ["FAIL_THRESHOLD"].lower() + mode = os.environ["SCAN_MODE"].lower() + severity_rank = {"critical": 4, "high": 3, "medium": 2, "low": 1, "unknown": 3} + threshold_rank = severity_rank[threshold] + scan_dir = Path(".tn-secret-scan") + config_path = Path(os.environ["GITLEAKS_CONFIG"]) + + rule_severities = {} + if config_path.exists(): + with config_path.open("rb") as config_file: + config = tomllib.load(config_file) + for rule in config.get("rules", []): + rule_id = rule.get("id") + if rule_id: + rule_severities[rule_id] = str(rule.get("severity", "high")).lower() + + findings = [] + + gitleaks_report = scan_dir / "gitleaks.json" + if gitleaks_report.exists() and gitleaks_report.stat().st_size: + with gitleaks_report.open("r", encoding="utf-8") as report: + try: + data = json.load(report) + except json.JSONDecodeError: + print("::error::Unable to parse Gitleaks JSON report") + sys.exit(2) + for item in data or []: + rule_id = str(item.get("RuleID") or "gitleaks-default") + severity = rule_severities.get(rule_id, "high") + findings.append({ + "engine": "gitleaks", + "severity": severity, + "rule": rule_id, + "detector": item.get("Description") or rule_id, + "file": item.get("File") or "unknown", + "line": item.get("StartLine") or "", + "commit": str(item.get("Commit") or "")[:12], + }) + + trufflehog_report = scan_dir / "trufflehog.jsonl" + if trufflehog_report.exists() and trufflehog_report.stat().st_size: + with trufflehog_report.open("r", encoding="utf-8") as report: + for line in report: + line = line.strip() + if not line: + continue + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + source = item.get("SourceMetadata", {}).get("Data", {}).get("Git", {}) + detector = item.get("DetectorName") or item.get("DetectorType") or "trufflehog" + findings.append({ + "engine": "trufflehog", + "severity": "high", + "rule": "verified-secret", + "detector": detector, + "file": source.get("file") or source.get("File") or "unknown", + "line": source.get("line") or source.get("Line") or "", + "commit": str(source.get("commit") or source.get("Commit") or "")[:12], + }) + + blocking = [ + finding for finding in findings + if severity_rank.get(finding["severity"], severity_rank["unknown"]) >= threshold_rank + ] + should_fail = mode in {"pr", "push"} and bool(blocking) + + summary_path = scan_dir / "summary.md" + with summary_path.open("w", encoding="utf-8") as summary: + summary.write("# TN Secret Scan Summary\n\n") + summary.write(f"- Mode: `{mode}`\n") + summary.write(f"- Fail threshold: `{threshold}`\n") + summary.write(f"- Total findings: `{len(findings)}`\n") + summary.write(f"- Blocking findings: `{len(blocking)}`\n") + summary.write("\n") + summary.write("Raw secret values are intentionally excluded. Confirmed secrets require rotation, revocation, migration, or abandonment; deleting the file is not sufficient remediation.\n\n") + if findings: + summary.write("| Engine | Severity | Rule | Location | Commit |\n") + summary.write("| --- | --- | --- | --- | --- |\n") + for finding in findings[:100]: + location = finding["file"] + if finding["line"]: + location = f"{location}:{finding['line']}" + row = [ + finding["engine"], + finding["severity"], + re.sub(r"[|\n\r]", " ", str(finding["rule"]))[:120], + re.sub(r"[|\n\r]", " ", str(location))[:180], + finding["commit"] or "-", + ] + summary.write("| " + " | ".join(row) + " |\n") + if len(findings) > 100: + summary.write(f"\nAdditional findings omitted from summary: {len(findings) - 100}\n") + + output_path = Path(os.environ["GITHUB_OUTPUT"]) + with output_path.open("a", encoding="utf-8") as output: + output.write(f"total_findings={len(findings)}\n") + output.write(f"blocking_findings={len(blocking)}\n") + output.write(f"should_fail={'true' if should_fail else 'false'}\n") + PY + + cat .tn-secret-scan/summary.md >> "$GITHUB_STEP_SUMMARY" - name: Check for Slack configuration - if: failure() + if: steps.summary.outputs.should_fail == 'true' id: slack-check run: | - if [ -n "${{ secrets.SLACK_BOT_TOKEN }}" ] && [ -n "${{ secrets.SLACK_CHANNEL_ID_GITHUB_NOTIFICATION }}" ]; then - echo "slack-configured=true" >> $GITHUB_OUTPUT + set -euo pipefail + if [ -n "${{ secrets.SLACK_BOT_TOKEN }}" ] && + [ -n "${{ secrets.SLACK_CHANNEL_ID_GITHUB_NOTIFICATION }}" ]; then + echo "slack-configured=true" >> "$GITHUB_OUTPUT" else - echo "slack-configured=false" >> $GITHUB_OUTPUT + echo "slack-configured=false" >> "$GITHUB_OUTPUT" fi - - name: Send notification to slack if secrets found - if: failure() && steps.slack-check.outputs.slack-configured == 'true' - uses: slackapi/slack-github-action@v1.26.0 - with: - channel-id: ${{ secrets.SLACK_CHANNEL_ID_GITHUB_NOTIFICATION }} - payload: | - { - "text": "TruffleHog scan detected secrets in ${{ - github.repository }}. Please review the action logs.", - "blocks": [ - { "type": "divider" }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "🚨 *Alert:* TruffleHog detected secrets in ${{ - github.repository }}. [View details](https://github.com/${{ - github.repository }}/actions/runs/${{ github.run_id }})" - } - } - ] - } + - name: Send sanitized Slack notification + if: steps.summary.outputs.should_fail == 'true' && steps.slack-check.outputs.slack-configured == 'true' env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID_GITHUB_NOTIFICATION }} + BLOCKING_FINDINGS: ${{ steps.summary.outputs.blocking_findings }} + TOTAL_FINDINGS: ${{ steps.summary.outputs.total_findings }} + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + + python3 <<'PY' > .tn-secret-scan/slack-payload.json + import json + import os + + text = ( + f"TN secret scan blocked {os.environ['REPOSITORY']}: " + f"{os.environ['BLOCKING_FINDINGS']} blocking findings " + f"({os.environ['TOTAL_FINDINGS']} total). " + "Raw secret values are not included. Confirmed secrets require rotation, revocation, migration, or abandonment." + ) + payload = { + "channel": os.environ["SLACK_CHANNEL_ID"], + "text": text, + "blocks": [ + {"type": "section", "text": {"type": "mrkdwn", "text": f"*TN secret scan failed* for `{os.environ['REPOSITORY']}`"}}, + {"type": "section", "text": {"type": "mrkdwn", "text": f"Blocking findings: `{os.environ['BLOCKING_FINDINGS']}`\nTotal findings: `{os.environ['TOTAL_FINDINGS']}`\n<{os.environ['RUN_URL']}|View GitHub Actions run>"}}, + {"type": "context", "elements": [{"type": "mrkdwn", "text": "Raw secret values are intentionally omitted. Rotate or revoke confirmed credentials."}]}, + ], + } + print(json.dumps(payload)) + PY + + curl -sSf \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H "Content-Type: application/json; charset=utf-8" \ + --data @.tn-secret-scan/slack-payload.json \ + https://slack.com/api/chat.postMessage > .tn-secret-scan/slack-response.json + + python3 <<'PY' + import json + + with open(".tn-secret-scan/slack-response.json", "r", encoding="utf-8") as response_file: + response = json.load(response_file) + if not response.get("ok"): + print(f"::warning::Slack notification failed: {response.get('error', 'unknown_error')}") + PY + + - name: Log missing Slack configuration + if: steps.summary.outputs.should_fail == 'true' && steps.slack-check.outputs.slack-configured == 'false' + run: | + echo "TN secret scan found blocking findings. Slack notification is not configured." + echo "Set SLACK_BOT_TOKEN and SLACK_CHANNEL_ID_GITHUB_NOTIFICATION to enable sanitized failure notifications." - - name: Log secrets detection without Slack - if: failure() && steps.slack-check.outputs.slack-configured == 'false' + - name: Enforce blocking findings + if: steps.summary.outputs.should_fail == 'true' run: | - echo "🚨 TruffleHog detected secrets in the repository!" - echo "Please review the scan results above and remediate any exposed secrets." - echo "Note: Slack notification is not configured. Set SLACK_BOT_TOKEN and SLACK_CHANNEL_ID_GITHUB_NOTIFICATION secrets to enable notifications." + echo "::error::TN secret scan found ${{ steps.summary.outputs.blocking_findings }} blocking findings." + echo "Raw secret values are intentionally omitted from logs and notifications." + echo "Confirmed secrets require rotation, revocation, migration, or abandonment." + exit 1 diff --git a/.github/workflows/secrets-scanning.yml b/.github/workflows/secrets-scanning.yml index 67c07a9..9bfff20 100644 --- a/.github/workflows/secrets-scanning.yml +++ b/.github/workflows/secrets-scanning.yml @@ -1,15 +1,31 @@ -name: Caller TruffleHog Scan +name: Caller TN Secret Scan on: + pull_request: push: branches: - - fixed + - main + - develop + workflow_dispatch: + inputs: + scan_mode: + description: "Use history for manual inventory scans." + required: true + default: history + type: choice + options: + - history + - pr + - push jobs: trigger_scan: - uses: storyprotocol/gha-workflows/.github/workflows/reusable-secrets-scanning.yml@main + uses: treasurenetprotocol/reusable-workflows/.github/workflows/reusable-secrets-scanning.yml@main with: - branch: ${{ github.ref_name }} + scan_mode: ${{ github.event_name == 'workflow_dispatch' && inputs.scan_mode || github.event_name == 'pull_request' && 'pr' || 'push' }} + fetch_depth: 0 + fail_threshold: high + config_ref: main secrets: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} SLACK_CHANNEL_ID_GITHUB_NOTIFICATION: ${{ secrets.SLACK_CHANNEL_ID_GITHUB_NOTIFICATION }} diff --git a/README.md b/README.md index 2fa90af..d225708 100644 --- a/README.md +++ b/README.md @@ -9,4 +9,71 @@ | Asset | Description | Usage | | --------------- | --------------- | --------------- | | `docs/security/secret-taxonomy.md` | TN-wide secret taxonomy, severity model, and remediation rules | Read before tuning secret-scanning workflows | -| `security/gitleaks/tn-gitleaks.toml` | Shared Gitleaks rules for TN-specific and general software-engineering secrets | Use from reusable secret-scanning workflows | +| `security/gitleaks/tn-gitleaks.toml` | Shared Gitleaks rules for TN-specific and general software-engineering secrets | Used by `reusable-secrets-scanning.yml` | +| `.github/workflows/reusable-secrets-scanning.yml` | Reusable TN secret scan workflow that runs Gitleaks and TruffleHog | Call from TN repositories for PR, push, or manual history scans | + +## Reusable Secret Scanning + +`reusable-secrets-scanning.yml` runs both scanner engines: + +- Gitleaks with `security/gitleaks/tn-gitleaks.toml` for TN-specific blocking rules. +- TruffleHog with verified-secret detection enabled. + +Supported scan modes: + +- `pr`: scans pull request changes and fails on findings at or above `fail_threshold`. +- `push`: scans new pushed commits and fails on findings at or above `fail_threshold`. +- `history`: scans available repository history for inventory and does not block by default. + +The workflow intentionally does not print raw secret values in logs, step summaries, or Slack notifications. Summaries include scanner engine, severity, rule, path, line, and commit where available. Confirmed secrets require rotation, revocation, migration, or abandonment; deleting a file or rewriting history is not sufficient remediation by itself. + +### Minimal Caller + +```yaml +name: TN Secret Scan + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + inputs: + scan_mode: + description: "Use history for manual inventory scans." + required: true + default: history + type: choice + options: + - history + - pr + - push + +jobs: + secret_scan: + uses: treasurenetprotocol/reusable-workflows/.github/workflows/reusable-secrets-scanning.yml@main + with: + scan_mode: ${{ github.event_name == 'workflow_dispatch' && inputs.scan_mode || github.event_name == 'pull_request' && 'pr' || 'push' }} + fetch_depth: 0 + fail_threshold: high + config_ref: main + secrets: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + SLACK_CHANNEL_ID_GITHUB_NOTIFICATION: ${{ secrets.SLACK_CHANNEL_ID_GITHUB_NOTIFICATION }} +``` + +Slack secrets are optional. When both `SLACK_BOT_TOKEN` and `SLACK_CHANNEL_ID_GITHUB_NOTIFICATION` are present, failures send a sanitized notification with counts and a link to the GitHub Actions run. + +### Inputs + +| Input | Default | Description | +| --- | --- | --- | +| `scan_mode` | `pr` | `pr`, `push`, or `history`. | +| `fetch_depth` | `0` | Checkout depth. Use `0` for complete history and accurate range scans. | +| `fail_threshold` | `high` | Lowest severity that blocks PR/push scans: `critical`, `high`, `medium`, or `low`. | +| `config_ref` | `main` | Ref in `treasurenetprotocol/reusable-workflows` used to fetch the shared Gitleaks config. Pin this to a reviewed tag or commit for stricter rollout control. | +| `config_path` | `security/gitleaks/tn-gitleaks.toml` | Gitleaks config path inside this repository. | +| `gitleaks_version` | `v8.27.2` | Gitleaks container version. | +| `trufflehog_version` | `3.90.5` | TruffleHog container version. | +| `branch` | empty | Optional legacy ref override. Prefer the event ref. | +| `depth` | `2` | Deprecated legacy input retained so old callers do not break validation. | From 9b1df17edc50b51f8f9f7b4251e8edd90e1f8252 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 28 Jun 2026 09:49:34 +0000 Subject: [PATCH 2/5] fix: validate gitleaks runtime compatibility INF-162 --- .github/workflows/reusable-secrets-scanning.yml | 2 +- security/gitleaks/tn-gitleaks.toml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable-secrets-scanning.yml b/.github/workflows/reusable-secrets-scanning.yml index ac269ff..3799e4d 100644 --- a/.github/workflows/reusable-secrets-scanning.yml +++ b/.github/workflows/reusable-secrets-scanning.yml @@ -157,12 +157,12 @@ jobs: args=( git - --source /repo --config "/repo/${GITLEAKS_CONFIG}" --report-format json --report-path /repo/.tn-secret-scan/gitleaks.json --redact --exit-code 0 + /repo ) if [ -n "${{ steps.scan-range.outputs.gitleaks_log_opts }}" ]; then diff --git a/security/gitleaks/tn-gitleaks.toml b/security/gitleaks/tn-gitleaks.toml index a284865..51bd2c3 100644 --- a/security/gitleaks/tn-gitleaks.toml +++ b/security/gitleaks/tn-gitleaks.toml @@ -52,7 +52,7 @@ keywords = [ id = "tn-wallet-keystore-json" description = "Wallet keystore JSON containing encrypted private key material" severity = "critical" -regex = '''(?i)"crypto"\s*:\s*\{[\s\S]{0,1200}"ciphertext"\s*:\s*"[a-f0-9]{64,}"''' +regex = '''(?i)"crypto"\s*:\s*\{[\s\S]{0,1000}"ciphertext"\s*:\s*"[a-f0-9]{64,}"''' keywords = [ "ciphertext", "crypto", @@ -92,7 +92,7 @@ keywords = [ id = "tn-gcp-service-account-private-key" description = "GCP service account private key JSON" severity = "critical" -regex = '''(?s)"type"\s*:\s*"service_account".{0,2000}"private_key"\s*:\s*"-----BEGIN PRIVATE KEY-----''' +regex = '''(?s)"type"\s*:\s*"service_account".{0,1000}"private_key"\s*:\s*"-----BEGIN PRIVATE KEY-----''' keywords = [ "service_account", "private_key" @@ -122,7 +122,7 @@ keywords = [ id = "tn-github-app-private-key" description = "GitHub App private key" severity = "critical" -regex = '''(?s)-----BEGIN RSA PRIVATE KEY-----.{0,2000}github.{0,2000}app''' +regex = '''(?s)-----BEGIN RSA PRIVATE KEY-----.{0,1000}github.{0,1000}app''' keywords = [ "github", "PRIVATE KEY" From f06c32486b026248f47a2c3343b28ce4e2d6046d Mon Sep 17 00:00:00 2001 From: Dongchen Xie Date: Tue, 14 Jul 2026 11:20:30 -0700 Subject: [PATCH 3/5] fix: export sanitized secret inventory INF-162 --- .../workflows/reusable-secrets-scanning.yml | 47 +++++++++++++++++-- README.md | 2 +- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable-secrets-scanning.yml b/.github/workflows/reusable-secrets-scanning.yml index 3799e4d..5ab8ce6 100644 --- a/.github/workflows/reusable-secrets-scanning.yml +++ b/.github/workflows/reusable-secrets-scanning.yml @@ -209,6 +209,7 @@ jobs: set -euo pipefail python3 <<'PY' + import html import json import os import re @@ -253,6 +254,7 @@ jobs: "file": item.get("File") or "unknown", "line": item.get("StartLine") or "", "commit": str(item.get("Commit") or "")[:12], + "verified": None, }) trufflehog_report = scan_dir / "trufflehog.jsonl" @@ -276,6 +278,7 @@ jobs: "file": source.get("file") or source.get("File") or "unknown", "line": source.get("line") or source.get("Line") or "", "commit": str(source.get("commit") or source.get("Commit") or "")[:12], + "verified": True, }) blocking = [ @@ -284,6 +287,28 @@ jobs: ] should_fail = mode in {"pr", "push"} and bool(blocking) + inventory_path = scan_dir / "sanitized-findings.json" + with inventory_path.open("w", encoding="utf-8") as inventory_file: + json.dump( + { + "schema_version": 1, + "mode": mode, + "fail_threshold": threshold, + "total_findings": len(findings), + "blocking_findings": len(blocking), + "findings": findings, + }, + inventory_file, + indent=2, + sort_keys=True, + ) + inventory_file.write("\n") + + def markdown_cell(value, limit): + value = re.sub(r"[\x00-\x1f\x7f]", " ", str(value)) + value = html.escape(value, quote=True).replace("|", "|") + return value[:limit] + summary_path = scan_dir / "summary.md" with summary_path.open("w", encoding="utf-8") as summary: summary.write("# TN Secret Scan Summary\n\n") @@ -301,11 +326,11 @@ jobs: if finding["line"]: location = f"{location}:{finding['line']}" row = [ - finding["engine"], - finding["severity"], - re.sub(r"[|\n\r]", " ", str(finding["rule"]))[:120], - re.sub(r"[|\n\r]", " ", str(location))[:180], - finding["commit"] or "-", + markdown_cell(finding["engine"], 40), + markdown_cell(finding["severity"], 20), + markdown_cell(finding["rule"], 120), + markdown_cell(location, 180), + markdown_cell(finding["commit"] or "-", 40), ] summary.write("| " + " | ".join(row) + " |\n") if len(findings) > 100: @@ -320,6 +345,18 @@ jobs: cat .tn-secret-scan/summary.md >> "$GITHUB_STEP_SUMMARY" + - name: Upload sanitized scan inventory + if: always() && steps.summary.outcome == 'success' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: tn-secret-scan-${{ github.run_id }} + path: | + .tn-secret-scan/summary.md + .tn-secret-scan/sanitized-findings.json + if-no-files-found: error + retention-days: 30 + include-hidden-files: true + - name: Check for Slack configuration if: steps.summary.outputs.should_fail == 'true' id: slack-check diff --git a/README.md b/README.md index d225708..2d752a6 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Supported scan modes: - `push`: scans new pushed commits and fails on findings at or above `fail_threshold`. - `history`: scans available repository history for inventory and does not block by default. -The workflow intentionally does not print raw secret values in logs, step summaries, or Slack notifications. Summaries include scanner engine, severity, rule, path, line, and commit where available. Confirmed secrets require rotation, revocation, migration, or abandonment; deleting a file or rewriting history is not sufficient remediation by itself. +The workflow intentionally does not print raw secret values in logs, step summaries, artifacts, or Slack notifications. Summaries include scanner engine, severity, rule, path, line, and commit where available. Every successful scan also uploads a 30-day `tn-secret-scan-` artifact containing the Markdown summary and the complete sanitized JSON inventory. Raw Gitleaks and TruffleHog reports are never uploaded. Confirmed secrets require rotation, revocation, migration, or abandonment; deleting a file or rewriting history is not sufficient remediation by itself. ### Minimal Caller From 97aaee06fa70c5a31edf567fb81aac3e1ec96407 Mon Sep 17 00:00:00 2001 From: Dongchen Xie Date: Tue, 14 Jul 2026 11:29:41 -0700 Subject: [PATCH 4/5] fix: constrain incremental secret scan refs INF-162 --- .../workflows/reusable-secrets-scanning.yml | 81 ++++++++++++++++--- README.md | 2 + 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/.github/workflows/reusable-secrets-scanning.yml b/.github/workflows/reusable-secrets-scanning.yml index 5ab8ce6..a95c5d1 100644 --- a/.github/workflows/reusable-secrets-scanning.yml +++ b/.github/workflows/reusable-secrets-scanning.yml @@ -70,6 +70,8 @@ jobs: PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} PUSH_BEFORE_SHA: ${{ github.event.before }} PUSH_AFTER_SHA: ${{ github.sha }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + REF_NAME: ${{ github.ref_name }} steps: - name: Validate inputs run: | @@ -92,13 +94,13 @@ jobs: esac - name: Checkout caller repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #v4.1.4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs.branch }} fetch-depth: ${{ inputs.fetch_depth }} - name: Checkout TN scanner configuration - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #v4.1.4 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: treasurenetprotocol/reusable-workflows ref: ${{ inputs.config_ref }} @@ -115,25 +117,76 @@ jobs: gitleaks_log_opts="" trufflehog_since_commit="" + trufflehog_max_depth="" + trufflehog_skip_additional_refs="false" + + set_single_commit_range() { + local target="$1" + gitleaks_log_opts="-1 ${target}" + trufflehog_max_depth="1" + } + + set_commit_range() { + local base="$1" + local target="$2" + gitleaks_log_opts="${base}..${target}" + trufflehog_since_commit="${base}" + } + + find_default_branch_merge_base() { + local ref + local merge_base + + if [ -z "${DEFAULT_BRANCH}" ] || [ "${REF_NAME}" = "${DEFAULT_BRANCH}" ]; then + return 0 + fi + + for ref in \ + "refs/remotes/origin/${DEFAULT_BRANCH}" \ + "origin/${DEFAULT_BRANCH}" \ + "refs/heads/${DEFAULT_BRANCH}"; do + if git cat-file -e "${ref}^{commit}" 2>/dev/null; then + merge_base="$(git merge-base HEAD "${ref}" 2>/dev/null || true)" + if [ -n "${merge_base}" ]; then + printf '%s' "${merge_base}" + return 0 + fi + fi + done + } case "${SCAN_MODE}" in pr) if [ -n "${PR_BASE_SHA}" ] && git cat-file -e "${PR_BASE_SHA}^{commit}" 2>/dev/null; then - gitleaks_log_opts="${PR_BASE_SHA}..${PR_HEAD_SHA:-HEAD}" - trufflehog_since_commit="${PR_BASE_SHA}" + set_commit_range "${PR_BASE_SHA}" "${PR_HEAD_SHA:-HEAD}" else - echo "::warning::Pull request base SHA is unavailable in this checkout; scanning the checked-out history." + fallback_base="$(find_default_branch_merge_base)" + if [ -n "${fallback_base}" ]; then + echo "::warning::Pull request base SHA is unavailable; using merge-base ${fallback_base:0:12} with ${DEFAULT_BRANCH}." + set_commit_range "${fallback_base}" HEAD + else + echo "::warning::Pull request base SHA and a default-branch merge-base are unavailable; scanning HEAD only." + set_single_commit_range HEAD + fi fi + trufflehog_skip_additional_refs="true" ;; push) if [ -n "${PUSH_BEFORE_SHA}" ] && \ [ "${PUSH_BEFORE_SHA}" != "0000000000000000000000000000000000000000" ] && \ git cat-file -e "${PUSH_BEFORE_SHA}^{commit}" 2>/dev/null; then - gitleaks_log_opts="${PUSH_BEFORE_SHA}..${PUSH_AFTER_SHA:-HEAD}" - trufflehog_since_commit="${PUSH_BEFORE_SHA}" + set_commit_range "${PUSH_BEFORE_SHA}" "${PUSH_AFTER_SHA:-HEAD}" else - echo "::warning::Push before SHA is unavailable in this checkout; scanning the checked-out history." + fallback_base="$(find_default_branch_merge_base)" + if [ -n "${fallback_base}" ]; then + echo "::warning::Push before SHA is unavailable; using merge-base ${fallback_base:0:12} with ${DEFAULT_BRANCH}." + set_commit_range "${fallback_base}" "${PUSH_AFTER_SHA:-HEAD}" + else + echo "::warning::Push before SHA and a default-branch merge-base are unavailable; scanning HEAD only." + set_single_commit_range "${PUSH_AFTER_SHA:-HEAD}" + fi fi + trufflehog_skip_additional_refs="true" ;; history) echo "History mode scans the checked-out repository history and does not block by default." @@ -143,6 +196,8 @@ jobs: { echo "gitleaks_log_opts=${gitleaks_log_opts}" echo "trufflehog_since_commit=${trufflehog_since_commit}" + echo "trufflehog_max_depth=${trufflehog_max_depth}" + echo "trufflehog_skip_additional_refs=${trufflehog_skip_additional_refs}" } >> "$GITHUB_OUTPUT" - name: Run Gitleaks @@ -190,6 +245,14 @@ jobs: args+=(--since-commit "${{ steps.scan-range.outputs.trufflehog_since_commit }}") fi + if [ -n "${{ steps.scan-range.outputs.trufflehog_max_depth }}" ]; then + args+=(--max-depth "${{ steps.scan-range.outputs.trufflehog_max_depth }}") + fi + + if [ "${{ steps.scan-range.outputs.trufflehog_skip_additional_refs }}" = "true" ]; then + args+=(--skip-additional-refs) + fi + set +e docker run --rm \ -v "$GITHUB_WORKSPACE:/repo" \ @@ -347,7 +410,7 @@ jobs: - name: Upload sanitized scan inventory if: always() && steps.summary.outcome == 'success' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: tn-secret-scan-${{ github.run_id }} path: | diff --git a/README.md b/README.md index 2d752a6..de8ef1a 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ Supported scan modes: - `push`: scans new pushed commits and fails on findings at or above `fail_threshold`. - `history`: scans available repository history for inventory and does not block by default. +For a newly created branch where GitHub does not provide a usable `before` SHA, push mode derives the incremental range from the merge-base with the repository's default branch. If no merge-base exists, it scans only `HEAD`. PR and push modes skip unrelated refs; only history mode intentionally scans the complete fetched history. + The workflow intentionally does not print raw secret values in logs, step summaries, artifacts, or Slack notifications. Summaries include scanner engine, severity, rule, path, line, and commit where available. Every successful scan also uploads a 30-day `tn-secret-scan-` artifact containing the Markdown summary and the complete sanitized JSON inventory. Raw Gitleaks and TruffleHog reports are never uploaded. Confirmed secrets require rotation, revocation, migration, or abandonment; deleting a file or rewriting history is not sufficient remediation by itself. ### Minimal Caller From 18e0d3ac719858b53869693a78ecc4737f57f57e Mon Sep 17 00:00:00 2001 From: Dongchen Xie Date: Tue, 14 Jul 2026 11:37:06 -0700 Subject: [PATCH 5/5] fix: address secret scan review findings INF-162 --- .github/workflows/reusable-secrets-scanning.yml | 7 +++++-- .github/workflows/secrets-scanning.yml | 12 +----------- README.md | 14 +++----------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/.github/workflows/reusable-secrets-scanning.yml b/.github/workflows/reusable-secrets-scanning.yml index a95c5d1..3a13518 100644 --- a/.github/workflows/reusable-secrets-scanning.yml +++ b/.github/workflows/reusable-secrets-scanning.yml @@ -98,6 +98,7 @@ jobs: with: ref: ${{ inputs.branch }} fetch-depth: ${{ inputs.fetch_depth }} + persist-credentials: false - name: Checkout TN scanner configuration uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -106,6 +107,7 @@ jobs: ref: ${{ inputs.config_ref }} path: _tn-security-config fetch-depth: 1 + persist-credentials: false - name: Prepare scan range id: scan-range @@ -323,14 +325,15 @@ jobs: trufflehog_report = scan_dir / "trufflehog.jsonl" if trufflehog_report.exists() and trufflehog_report.stat().st_size: with trufflehog_report.open("r", encoding="utf-8") as report: - for line in report: + for line_number, line in enumerate(report, start=1): line = line.strip() if not line: continue try: item = json.loads(line) except json.JSONDecodeError: - continue + print(f"::error::Unable to parse TruffleHog JSONL record at line {line_number}") + sys.exit(2) source = item.get("SourceMetadata", {}).get("Data", {}).get("Git", {}) detector = item.get("DetectorName") or item.get("DetectorType") or "trufflehog" findings.append({ diff --git a/.github/workflows/secrets-scanning.yml b/.github/workflows/secrets-scanning.yml index 9bfff20..5fb8238 100644 --- a/.github/workflows/secrets-scanning.yml +++ b/.github/workflows/secrets-scanning.yml @@ -7,22 +7,12 @@ on: - main - develop workflow_dispatch: - inputs: - scan_mode: - description: "Use history for manual inventory scans." - required: true - default: history - type: choice - options: - - history - - pr - - push jobs: trigger_scan: uses: treasurenetprotocol/reusable-workflows/.github/workflows/reusable-secrets-scanning.yml@main with: - scan_mode: ${{ github.event_name == 'workflow_dispatch' && inputs.scan_mode || github.event_name == 'pull_request' && 'pr' || 'push' }} + scan_mode: ${{ github.event_name == 'workflow_dispatch' && 'history' || github.event_name == 'pull_request' && 'pr' || 'push' }} fetch_depth: 0 fail_threshold: high config_ref: main diff --git a/README.md b/README.md index de8ef1a..9dfc180 100644 --- a/README.md +++ b/README.md @@ -40,22 +40,12 @@ on: branches: - main workflow_dispatch: - inputs: - scan_mode: - description: "Use history for manual inventory scans." - required: true - default: history - type: choice - options: - - history - - pr - - push jobs: secret_scan: uses: treasurenetprotocol/reusable-workflows/.github/workflows/reusable-secrets-scanning.yml@main with: - scan_mode: ${{ github.event_name == 'workflow_dispatch' && inputs.scan_mode || github.event_name == 'pull_request' && 'pr' || 'push' }} + scan_mode: ${{ github.event_name == 'workflow_dispatch' && 'history' || github.event_name == 'pull_request' && 'pr' || 'push' }} fetch_depth: 0 fail_threshold: high config_ref: main @@ -64,6 +54,8 @@ jobs: SLACK_CHANNEL_ID_GITHUB_NOTIFICATION: ${{ secrets.SLACK_CHANNEL_ID_GITHUB_NOTIFICATION }} ``` +Manual dispatch always runs non-blocking `history` mode. PR and push modes require their native GitHub event payloads so the reusable workflow can derive an accurate incremental commit range. + Slack secrets are optional. When both `SLACK_BOT_TOKEN` and `SLACK_CHANNEL_ID_GITHUB_NOTIFICATION` are present, failures send a sanitized notification with counts and a link to the GitHub Actions run. ### Inputs