diff --git a/.gitignore b/.gitignore index f49e1b4..57045ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,71 @@ +# ============================================================================ +# AutoJSON Specific Files +# ============================================================================ # AutoJSON output files .autojson_debug_context.json .autojson_filter_config.json dashboard/.service_config.json +# AutoJSON time-travel archives +.autojson_history/ + +# AutoJSON temporary files +*.tmp +.autojson_tmp_*.json + +# Test outputs +test_*.json +*_demo.json + +# ============================================================================ +# Sensitive Data & Secrets (CRITICAL - Never commit these!) +# ============================================================================ +# Environment variables +.env +.env.* +!.env.example +.envrc + +# API Keys and tokens +*.key +*.pem +*.p12 +*.pfx +*.cer +*.crt +*.der +secrets/ +.secrets/ + +# Configuration files with credentials +*secret*.json +*credentials*.json +*config*.local.* +*private*.json + +# Database files +*.db +*.sqlite +*.sqlite3 + +# Backup files +*.backup +*.bak +*.old +*.orig + +# ============================================================================ # Python +# ============================================================================ +# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class + +# C extensions *.so + +# Distribution / packaging .Python build/ develop-eggs/ @@ -28,39 +86,271 @@ share/python-wheels/ *.egg MANIFEST -# Virtual environments -venv/ -env/ -ENV/ -env.bak/ -venv.bak/ +# PyInstaller +*.manifest +*.spec -# Testing -.pytest_cache/ -.coverage +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports htmlcov/ .tox/ -.hypothesis/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml *.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: *.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +docs/.doctrees/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints +*.ipynb + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# PEP 582 +__pypackages__/ -# IDEs +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# ============================================================================ +# IDEs and Editors +# ============================================================================ +# VSCode .vscode/ +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# PyCharm / IntelliJ IDEA .idea/ +.idea/* +*.iml +*.ipr +*.iws +.idea_modules/ + +# Sublime Text +*.sublime-project +*.sublime-workspace +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache +*.sublime-workspace +sftp-config.json +sftp-config-alt*.json + +# Vim *.swp *.swo +.vim/ +.vimsettings +tags +.netrwhist + +# Emacs +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* + +# TextMate +*.tmproj +*.tmproject +tmtags + +# Eclipse +.project +.cproject +.classpath +.settings/ +.buildpath + +# NetBeans +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ + +# ============================================================================ +# Operating System Files +# ============================================================================ +# Common to all OS *~ + +# macOS .DS_Store +.AppleDouble +.LSOverride +._* +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk -# Temporary files -*.tmp -.autojson_tmp_*.json +# Windows +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db +*.stackdump +[Dd]esktop.ini +$RECYCLE.BIN/ +*.cab +*.msi +*.msix +*.msm +*.msp +*.lnk + +# Linux +.directory +.Trash-* +.nfs* + +# ============================================================================ +# Temporary and Cache Files +# ============================================================================ tmp/ temp/ +*.tmp +*.temp +*.cache +.cache/ -# Test outputs -test_*.json -*_demo.json +# ============================================================================ +# Logs +# ============================================================================ +*.log +logs/ +*.log.* +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* -# AutoJSON time-travel archives -.autojson_history/ +# ============================================================================ +# Node.js / JavaScript (if applicable for dashboard or extensions) +# ============================================================================ +node_modules/ +npm-debug.log +yarn-error.log +.npm +.yarn-integrity +package-lock.json +yarn.lock +.pnp +.pnp.js +.eslintcache + +# ============================================================================ +# Misc +# ============================================================================ +# Archives +*.zip +*.tar +*.tar.gz +*.tgz +*.rar +*.7z + +# Docker +docker-compose.override.yml +.dockerignore + +# Git +*.rej +.git-rewrite/ + +# Other +.history/ diff --git a/README.md b/README.md index 1b564b1..e5d0905 100644 --- a/README.md +++ b/README.md @@ -397,6 +397,9 @@ indicate security vulnerabilities in auth, payment, or admin endpoints. - **[FILE_ANCHORS_GUIDE.md](FILE_ANCHORS_GUIDE.md)** - File navigation implementation - **[BATCHING_SUMMARY.md](BATCHING_SUMMARY.md)** - Performance optimization details +### Security +- **[SECURITY.md](SECURITY.md)** - šŸ”’ Security policy, vulnerability reporting, and best practices + --- ## šŸ”’ Security & Privacy @@ -406,6 +409,14 @@ indicate security vulnerabilities in auth, payment, or admin endpoints. - āœ… **No Internet Required** - Fully offline capable - āš ļø **Sensitive Data** - `.autojson_debug_context.json` may contain error messages with user data +**Security Resources:** +- **[SECURITY.md](SECURITY.md)** - šŸ”’ Complete security policy, vulnerability reporting, and best practices +- **Git History Scanner** - Scan for accidentally committed secrets: + ```bash + python scripts/scan_git_history.py + python scripts/scan_git_history.py --output security_report.txt + ``` + **Recommended `.gitignore`:** ``` .autojson_debug_context.json diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..a2f4fe6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,257 @@ +# Security Policy + +## Overview + +AutoJSON is a local error aggregation system designed with security and privacy in mind. This document outlines our security practices, how to report vulnerabilities, and our commitment to maintaining a secure open-source project. + +## Security Principles + +### šŸ”’ Privacy-First Design +- **100% Local Processing**: All error data is processed and stored locally on your machine +- **No External Services**: AutoJSON does not send data to external servers or third-party services +- **Localhost Only**: The aggregator service runs exclusively on `localhost:9000` +- **Offline Capable**: Fully functional without internet connectivity + +### āš ļø Data Considerations +- Error messages and stack traces may contain sensitive information (user data, API keys, etc.) +- The `.autojson_debug_context.json` file should be treated as potentially sensitive +- Configuration files may contain domain names and endpoint patterns +- All AutoJSON output files are excluded from version control by default + +## Supported Versions + +We provide security updates for the following versions: + +| Version | Supported | +| ------- | ------------------ | +| Latest | :white_check_mark: | +| < 1.0 | :x: | + +## Reporting a Vulnerability + +We take security vulnerabilities seriously. If you discover a security issue, please follow these steps: + +### 1. **Do NOT** Open a Public Issue +Security vulnerabilities should not be disclosed publicly until a fix is available. + +### 2. Report Privately +Please report security vulnerabilities through one of these channels: + +- **GitHub Security Advisories**: [Report a vulnerability](https://github.com/igeorgegabriel/AutoJSON/security/advisories/new) +- **Email**: Create an issue with the title "SECURITY" and we'll provide a secure contact method + +### 3. Include Details +Please provide: +- Description of the vulnerability +- Steps to reproduce the issue +- Potential impact +- Any suggested fixes (if available) +- Your contact information for follow-up + +### 4. Response Timeline +- **Initial Response**: Within 48 hours +- **Status Update**: Within 7 days +- **Fix Timeline**: Depends on severity (critical issues prioritized) + +### 5. Disclosure Policy +- We will acknowledge your report within 48 hours +- We will provide a detailed response within 7 days +- We will work with you to understand and fix the issue +- We will credit you in the security advisory (unless you prefer to remain anonymous) +- We will coordinate public disclosure after a fix is available + +## Security Best Practices + +### For Users + +#### 1. **Protect Your Output Files** +```bash +# Ensure these files are in .gitignore +.autojson_debug_context.json +.autojson_filter_config.json +.autojson_history/ +``` + +#### 2. **Review Error Data Before Sharing** +- Error messages may contain sensitive data (tokens, user info, API keys) +- Review `.autojson_debug_context.json` before sharing with LLMs or team members +- Consider sanitizing data in production environments + +#### 3. **Secure Your Configuration** +```json +{ + "frontend": { + "domain_whitelist": ["localhost", "yourdomain.com"], + "message_blacklist": ["API_KEY=", "token=", "password="] + } +} +``` + +#### 4. **Run Locally Only** +- Never expose the AutoJSON service (port 9000) to the internet +- Use firewall rules to restrict access to localhost +- Do not bind to `0.0.0.0` or public IP addresses + +#### 5. **Keep Dependencies Updated** +```bash +# Regularly update dependencies to get security patches +pip install --upgrade -r requirements.txt +``` + +#### 6. **Use Virtual Environments** +```bash +# Isolate AutoJSON dependencies +python -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +pip install -r requirements.txt +``` + +### For Contributors + +#### 1. **Never Commit Secrets** +- API keys, tokens, passwords +- Personal data or PII +- Production credentials +- Real error logs with sensitive data + +#### 2. **Use Test Data** +- Create sanitized test fixtures +- Use placeholder values for tests +- Generate synthetic error data + +#### 3. **Code Review Checklist** +- [ ] No hardcoded credentials +- [ ] No sensitive data in tests +- [ ] Input validation for all endpoints +- [ ] Proper error handling +- [ ] No arbitrary code execution risks +- [ ] Dependencies are pinned and trusted + +#### 4. **Security Testing** +```bash +# Run tests before committing +pytest tests/ -v + +# Check for common security issues +pip install bandit safety +bandit -r autojson_core/ +safety check +``` + +## Known Security Considerations + +### 1. **Local File System Access** +- AutoJSON writes to local files (`.autojson_debug_context.json`) +- Ensure proper file permissions in your environment +- Files are created with default user permissions + +### 2. **Error Message Content** +- Error messages may inadvertently contain sensitive data +- Users should configure filters to exclude sensitive patterns +- Consider sanitizing error messages in production + +### 3. **Cross-Origin Requests** +- Chrome extension captures errors from web pages +- CORS policies prevent access to cross-origin script contents +- This is a security feature, not a bug + +### 4. **Stack Traces** +- Backend stack traces may reveal code structure +- This is intended for debugging but should be protected +- Do not expose `.autojson_debug_context.json` publicly + +### 5. **LLM Integration** +- Be cautious when sharing error data with external LLM services +- Consider data residency and privacy policies +- Use local LLM solutions for sensitive projects + +## Security Updates + +Security updates will be: +- Released as soon as possible after discovery +- Announced in the repository's Security Advisories +- Documented in CHANGELOG.md +- Tagged with version bumps according to severity + +## Dependency Security + +AutoJSON uses minimal dependencies to reduce attack surface: + +### Core Dependencies +- `fastapi` - Web framework (regularly updated) +- `uvicorn` - ASGI server (security-focused) +- `pydantic` - Data validation (prevents injection attacks) +- `httpx` - HTTP client (actively maintained) + +### Monitoring +We monitor dependencies for vulnerabilities using: +- GitHub Dependabot +- Security advisories +- Community reports + +### Update Policy +- Critical vulnerabilities: Immediate update +- High severity: Update within 7 days +- Medium/Low: Update in next release + +## Vulnerability History + +No security vulnerabilities have been reported or discovered to date. + +When vulnerabilities are disclosed, they will be listed here with: +- CVE identifier (if applicable) +- Severity level +- Affected versions +- Fixed versions +- Mitigation steps + +## Compliance + +### Data Privacy +- **GDPR**: AutoJSON processes data locally; users control all data +- **CCPA**: No data collection or sale +- **HIPAA/PCI-DSS**: Not certified; users responsible for sanitizing sensitive data + +### Open Source License +- MIT License - see [LICENSE](LICENSE) +- No warranty or liability for security issues +- Use at your own risk in production environments + +## Security Checklist for Production Use + +If you plan to use AutoJSON in a production environment: + +- [ ] Review and sanitize all error messages +- [ ] Configure strict filters to exclude sensitive patterns +- [ ] Ensure `.autojson_debug_context.json` is not committed to version control +- [ ] Restrict service access to localhost only +- [ ] Use HTTPS for any remote LLM integrations +- [ ] Implement log rotation for error archives +- [ ] Regular security audits of error data +- [ ] Keep dependencies updated +- [ ] Monitor for unusual error patterns +- [ ] Have an incident response plan + +## Contact + +- **Security Issues**: [GitHub Security Advisories](https://github.com/igeorgegabriel/AutoJSON/security/advisories) +- **General Issues**: [GitHub Issues](https://github.com/igeorgegabriel/AutoJSON/issues) +- **Discussions**: [GitHub Discussions](https://github.com/igeorgegabriel/AutoJSON/discussions) + +## Acknowledgments + +We thank the security research community for helping keep AutoJSON secure. Security researchers who responsibly disclose vulnerabilities will be acknowledged in our security advisories (with permission). + +--- + +**Last Updated**: February 2026 +**Version**: 1.0 + +--- + +## Additional Resources + +- [OWASP Top 10](https://owasp.org/www-project-top-ten/) +- [Python Security Best Practices](https://python.readthedocs.io/en/stable/library/security_warnings.html) +- [FastAPI Security](https://fastapi.tiangolo.com/tutorial/security/) +- [GitHub Security Best Practices](https://docs.github.com/en/code-security) diff --git a/SECURITY_QUICK_REFERENCE.md b/SECURITY_QUICK_REFERENCE.md new file mode 100644 index 0000000..e22a411 --- /dev/null +++ b/SECURITY_QUICK_REFERENCE.md @@ -0,0 +1,111 @@ +# Security Quick Reference + +Quick commands for keeping your AutoJSON repository secure. + +## šŸ” Scan Git History for Secrets + +```bash +# Quick scan +python scripts/scan_git_history.py + +# Save detailed report +python scripts/scan_git_history.py --output security_report.txt + +# Verbose mode +python scripts/scan_git_history.py --verbose +``` + +## šŸ“‹ What Gets Scanned + +### Sensitive Files Detected: +- `.env`, `.key`, `.pem`, `.p12`, `.pfx` files +- Database files (`.db`, `.sqlite`) +- Backup files (`.bak`, `.backup`, `.old`) +- Files with patterns: `secret`, `credential`, `password`, `token` + +### Secret Patterns Detected: +- API Keys and Secret Keys +- Passwords and Tokens +- Bearer Tokens +- Private Keys (RSA, etc.) +- Database Connection Strings (MongoDB, PostgreSQL, MySQL, Redis) +- Cloud Provider Keys (AWS, Azure, GCP) +- Service Tokens (GitHub, GitLab, Slack, Stripe) + +## šŸ›”ļø Best Practices + +### Before Committing +```bash +# Check what files will be committed +git status + +# Review the .gitignore file +cat .gitignore | grep -E "env|secret|key|token" + +# Scan for any accidentally staged secrets +python scripts/scan_git_history.py +``` + +### Before Open-Sourcing +```bash +# Full security scan with report +python scripts/scan_git_history.py --output pre-release-security-scan.txt + +# Review the report +cat pre-release-security-scan.txt + +# Check .gitignore coverage +git ls-files | grep -E "\.(env|key|pem|secret)" +``` + +### Regular Audits +```bash +# Weekly/monthly security scan +python scripts/scan_git_history.py --verbose --output "security-scan-$(date +%Y-%m-%d).txt" +``` + +## šŸšØ If Secrets Are Found + +### Immediate Steps: +1. **Rotate the exposed credentials immediately** +2. **Review all findings** to confirm they are actual secrets +3. **Remove from git history** using one of these tools: + - `git-filter-repo` (recommended) + - BFG Repo-Cleaner + +### Using git-filter-repo: +```bash +# Install +pip install git-filter-repo + +# Remove specific file +git filter-repo --path path/to/secret.env --invert-paths + +# Force push (coordinate with team!) +git push origin --force --all +``` + +### Update .gitignore: +```bash +# Add pattern to prevent future commits +echo "*.env" >> .gitignore +echo "config/secrets/" >> .gitignore +git add .gitignore +git commit -m "Update .gitignore to exclude sensitive files" +``` + +## šŸ“š More Information + +- **Full Security Policy**: [SECURITY.md](SECURITY.md) +- **Contributing Guidelines**: [README.md](README.md#contributing) +- **All Scripts**: [scripts/README.md](scripts/README.md) + +## šŸ†˜ Need Help? + +- **Security Issues**: Use [GitHub Security Advisories](https://github.com/igeorgegabriel/AutoJSON/security/advisories) +- **General Questions**: [GitHub Discussions](https://github.com/igeorgegabriel/AutoJSON/discussions) +- **Bug Reports**: [GitHub Issues](https://github.com/igeorgegabriel/AutoJSON/issues) + +--- + +**Remember**: Security is everyone's responsibility. When in doubt, ask for help! diff --git a/scripts/README.md b/scripts/README.md index 8620c3e..e299dae 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -34,6 +34,46 @@ python scripts/time_travel.py list --- +## šŸ”’ Security Scripts + +### `scan_git_history.py` +Scan git repository history for accidentally committed sensitive files and secrets. + +**Usage:** +```bash +# Quick scan with summary +python scripts/scan_git_history.py + +# Verbose output with detailed progress +python scripts/scan_git_history.py --verbose + +# Save report to file +python scripts/scan_git_history.py --output security_report.txt +``` + +**What it detects:** +- šŸ”‘ API keys, tokens, passwords +- šŸ—„ļø Database connection strings +- šŸ” Private keys and certificates +- šŸ“„ Sensitive file extensions (.env, .key, .pem, etc.) +- āš ļø Files with suspicious patterns (secret, credential, etc.) + +**When to use:** +- Before open-sourcing your project +- Regular security audits +- After accidentally committing sensitive data +- When onboarding new contributors + +**Output:** +- Detailed report with findings by type +- Commit information (hash, author, date) +- Context and line numbers for secrets +- Recommendations for remediation + +**See:** [SECURITY.md](../SECURITY.md) for complete security policy and best practices. + +--- + ## Advanced Scripts ### `archive_errors.py` diff --git a/scripts/scan_git_history.py b/scripts/scan_git_history.py new file mode 100755 index 0000000..82f6b08 --- /dev/null +++ b/scripts/scan_git_history.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +""" +Git History Security Scanner for AutoJSON + +This script scans the git repository history to identify potentially leaked +sensitive files that should not have been committed. + +Usage: + python scripts/scan_git_history.py [--verbose] [--output report.txt] + +Features: +- Scans all commits in git history +- Identifies potentially sensitive files by extension and pattern +- Checks for common secrets patterns (API keys, tokens, passwords) +- Generates detailed report with recommendations +- Safe and read-only (does not modify repository) +""" + +import subprocess +import re +import argparse +import sys +from pathlib import Path +from typing import List, Dict, Set, Tuple +from datetime import datetime +from collections import defaultdict + + +class GitHistoryScanner: + """Scanner for detecting potentially leaked sensitive files in git history.""" + + # File extensions that commonly contain sensitive data + SENSITIVE_EXTENSIONS = { + '.env', '.key', '.pem', '.p12', '.pfx', '.cer', '.crt', '.der', + '.keystore', '.jks', '.secret', '.credentials', '.token', + '.db', '.sqlite', '.sqlite3', + '.backup', '.bak', '.old', '.orig' + } + + # Filename patterns that suggest sensitive content + SENSITIVE_PATTERNS = [ + r'secret', r'credential', r'password', r'passwd', r'token', + r'api[_-]?key', r'private', r'\.env', r'config\.local', + r'\.backup', r'\.bak', r'\.old' + ] + + # Content patterns that might indicate secrets + SECRET_PATTERNS = [ + (r'(?i)(api[_-]?key|apikey)\s*[:=]\s*["\']?([a-zA-Z0-9_\-]{20,})["\']?', 'API Key'), + (r'(?i)(secret[_-]?key|secretkey)\s*[:=]\s*["\']?([a-zA-Z0-9_\-]{20,})["\']?', 'Secret Key'), + (r'(?i)(password|passwd|pwd)\s*[:=]\s*["\']?([^\s"\']{8,})["\']?', 'Password'), + (r'(?i)(token)\s*[:=]\s*["\']?([a-zA-Z0-9_\-]{20,})["\']?', 'Token'), + (r'(?i)(bearer\s+[a-zA-Z0-9_\-\.]{20,})', 'Bearer Token'), + (r'-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----', 'Private Key'), + (r'(?i)mongodb(\+srv)?://[^\s]+', 'MongoDB Connection String'), + (r'(?i)postgres://[^\s]+', 'PostgreSQL Connection String'), + (r'(?i)mysql://[^\s]+', 'MySQL Connection String'), + (r'(?i)redis://[^\s]+', 'Redis Connection String'), + (r'(?i)aws_access_key_id\s*[:=]\s*["\']?([A-Z0-9]{20})["\']?', 'AWS Access Key'), + (r'(?i)aws_secret_access_key\s*[:=]\s*["\']?([a-zA-Z0-9/+]{40})["\']?', 'AWS Secret Key'), + (r'sk_live_[a-zA-Z0-9]{24,}', 'Stripe Live Key'), + (r'sk_test_[a-zA-Z0-9]{24,}', 'Stripe Test Key'), + (r'gh[pousr]_[a-zA-Z0-9]{36}', 'GitHub Token'), + (r'glpat-[a-zA-Z0-9\-_]{20,}', 'GitLab Token'), + (r'xox[baprs]-[a-zA-Z0-9-]{10,}', 'Slack Token'), + ] + + # Files that are safe to ignore + SAFE_FILES = { + '.gitignore', 'requirements.txt', 'package.json', 'package-lock.json', + 'Pipfile', 'poetry.lock', 'yarn.lock', 'LICENSE', 'README.md', + 'SECURITY.md', 'CHANGELOG.md', 'CONTRIBUTING.md', '.env.example' + } + + def __init__(self, verbose: bool = False): + self.verbose = verbose + self.findings: List[Dict] = [] + self.scanned_commits = 0 + self.scanned_files = 0 + + def log(self, message: str): + """Print message if verbose mode is enabled.""" + if self.verbose: + print(f"[INFO] {message}") + + def run_git_command(self, args: List[str]) -> str: + """Run a git command and return output.""" + try: + result = subprocess.run( + ['git'] + args, + capture_output=True, + text=True, + check=True + ) + return result.stdout + except subprocess.CalledProcessError as e: + print(f"[ERROR] Git command failed: {' '.join(args)}") + print(f"[ERROR] {e.stderr}") + return "" + + def get_all_commits(self) -> List[str]: + """Get list of all commit hashes.""" + output = self.run_git_command(['rev-list', '--all']) + commits = [line.strip() for line in output.split('\n') if line.strip()] + self.log(f"Found {len(commits)} commits to scan") + return commits + + def get_commit_files(self, commit: str) -> List[str]: + """Get list of files in a specific commit.""" + output = self.run_git_command(['ls-tree', '-r', '--name-only', commit]) + return [line.strip() for line in output.split('\n') if line.strip()] + + def get_file_content(self, commit: str, filepath: str) -> str: + """Get content of a file at a specific commit.""" + try: + output = self.run_git_command(['show', f'{commit}:{filepath}']) + return output + except Exception: + return "" + + def is_sensitive_filename(self, filename: str) -> Tuple[bool, str]: + """Check if filename suggests sensitive content.""" + basename = Path(filename).name.lower() + extension = Path(filename).suffix.lower() + + # Check extension + if extension in self.SENSITIVE_EXTENSIONS: + return True, f"Sensitive extension: {extension}" + + # Check patterns + for pattern in self.SENSITIVE_PATTERNS: + if re.search(pattern, basename, re.IGNORECASE): + return True, f"Sensitive pattern in filename: {pattern}" + + return False, "" + + def scan_content_for_secrets(self, content: str, filepath: str) -> List[Dict]: + """Scan file content for potential secrets.""" + secrets_found = [] + + # Skip binary files or very large files + if len(content) > 1_000_000 or '\x00' in content[:1000]: + return secrets_found + + for pattern, secret_type in self.SECRET_PATTERNS: + matches = re.finditer(pattern, content) + for match in matches: + # Get line number + line_num = content[:match.start()].count('\n') + 1 + + # Get context (the line where secret was found) + lines = content.split('\n') + if line_num <= len(lines): + context = lines[line_num - 1].strip()[:100] # First 100 chars + else: + context = match.group(0)[:100] + + secrets_found.append({ + 'type': secret_type, + 'file': filepath, + 'line': line_num, + 'context': context, + 'pattern': pattern + }) + + return secrets_found + + def scan_commit(self, commit: str) -> List[Dict]: + """Scan a single commit for sensitive files.""" + findings = [] + files = self.get_commit_files(commit) + + for filepath in files: + self.scanned_files += 1 + + # Skip safe files + basename = Path(filepath).name + if basename in self.SAFE_FILES: + continue + + # Check filename + is_sensitive, reason = self.is_sensitive_filename(filepath) + + if is_sensitive: + findings.append({ + 'commit': commit, + 'file': filepath, + 'reason': reason, + 'type': 'sensitive_filename' + }) + + # If file has sensitive name, also scan its content + content = self.get_file_content(commit, filepath) + if content: + secrets = self.scan_content_for_secrets(content, filepath) + for secret in secrets: + secret['commit'] = commit + secret['type'] = 'secret_in_content' + findings.append(secret) + else: + # For non-sensitive filenames, do a quick check for certain extensions + if filepath.endswith(('.json', '.txt', '.conf', '.config', '.ini', '.yaml', '.yml')): + content = self.get_file_content(commit, filepath) + if content: + secrets = self.scan_content_for_secrets(content, filepath) + for secret in secrets: + secret['commit'] = commit + secret['type'] = 'secret_in_content' + findings.append(secret) + + return findings + + def scan_repository(self): + """Scan entire repository history.""" + print("[*] Starting git history security scan...") + print("[*] This may take a while for large repositories...") + + commits = self.get_all_commits() + + for i, commit in enumerate(commits): + if i % 10 == 0: + print(f"[*] Scanning commit {i+1}/{len(commits)}...", end='\r') + + self.scanned_commits += 1 + commit_findings = self.scan_commit(commit) + self.findings.extend(commit_findings) + + print(f"\n[*] Scan complete!") + print(f"[*] Scanned {self.scanned_commits} commits and {self.scanned_files} files") + + def get_commit_info(self, commit: str) -> Dict: + """Get detailed information about a commit.""" + info = {} + + # Get commit date and author + output = self.run_git_command(['show', '-s', '--format=%ai|%an|%s', commit]) + if output: + parts = output.strip().split('|') + if len(parts) >= 3: + info['date'] = parts[0] + info['author'] = parts[1] + info['message'] = parts[2] + + return info + + def generate_report(self) -> str: + """Generate detailed security report.""" + report = [] + report.append("=" * 80) + report.append("GIT HISTORY SECURITY SCAN REPORT") + report.append("=" * 80) + report.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + report.append(f"Repository: {Path.cwd()}") + report.append(f"Commits Scanned: {self.scanned_commits}") + report.append(f"Files Scanned: {self.scanned_files}") + report.append(f"Potential Issues Found: {len(self.findings)}") + report.append("") + + if not self.findings: + report.append("āœ“ No sensitive files or secrets detected in git history.") + report.append("") + report.append("This is a good sign! Your repository appears clean.") + return "\n".join(report) + + # Group findings by type + by_type = defaultdict(list) + for finding in self.findings: + by_type[finding['type']].append(finding) + + # Report sensitive filenames + if by_type['sensitive_filename']: + report.append("=" * 80) + report.append("SENSITIVE FILENAMES DETECTED") + report.append("=" * 80) + report.append("") + + # Group by file + by_file = defaultdict(list) + for finding in by_type['sensitive_filename']: + by_file[finding['file']].append(finding) + + for filepath, findings_list in by_file.items(): + report.append(f"āš  File: {filepath}") + report.append(f" Reason: {findings_list[0]['reason']}") + report.append(f" Found in {len(findings_list)} commit(s)") + + # Show first few commits + for finding in findings_list[:3]: + info = self.get_commit_info(finding['commit']) + report.append(f" - Commit: {finding['commit'][:8]}") + if info: + report.append(f" Date: {info.get('date', 'Unknown')}") + report.append(f" Author: {info.get('author', 'Unknown')}") + + report.append("") + + # Report secrets in content + if by_type['secret_in_content']: + report.append("=" * 80) + report.append("POTENTIAL SECRETS DETECTED IN CONTENT") + report.append("=" * 80) + report.append("") + + # Group by secret type + by_secret_type = defaultdict(list) + for finding in by_type['secret_in_content']: + by_secret_type[finding.get('type', 'Unknown')].append(finding) + + for secret_type, findings_list in by_secret_type.items(): + report.append(f"āš  Secret Type: {secret_type}") + report.append(f" Occurrences: {len(findings_list)}") + report.append("") + + for finding in findings_list[:5]: # Show first 5 + report.append(f" File: {finding['file']}") + report.append(f" Line: {finding.get('line', 'Unknown')}") + report.append(f" Commit: {finding['commit'][:8]}") + report.append(f" Context: {finding.get('context', '')}") + report.append("") + + # Recommendations + report.append("=" * 80) + report.append("RECOMMENDATIONS") + report.append("=" * 80) + report.append("") + + if self.findings: + report.append("1. Review each finding to determine if it contains actual sensitive data") + report.append("") + report.append("2. If sensitive data is confirmed:") + report.append(" - Rotate any exposed credentials immediately") + report.append(" - Consider using git-filter-repo or BFG Repo-Cleaner to remove") + report.append(" sensitive data from git history") + report.append(" - Update .gitignore to prevent future commits") + report.append("") + report.append("3. Add patterns to .gitignore:") + report.append(" - Review SECURITY.md for recommended patterns") + report.append(" - Current .gitignore should already exclude common sensitive files") + report.append("") + report.append("4. For removing files from git history:") + report.append(" - Option A: git-filter-repo (recommended)") + report.append(" pip install git-filter-repo") + report.append(" git filter-repo --path --invert-paths") + report.append("") + report.append(" - Option B: BFG Repo-Cleaner") + report.append(" https://rtyley.github.io/bfg-repo-cleaner/") + report.append("") + report.append("āš  WARNING: Rewriting git history requires force-pushing!") + report.append(" Only do this if you're sure and have coordinated with your team.") + + report.append("") + report.append("=" * 80) + + return "\n".join(report) + + +def main(): + """Main entry point.""" + parser = argparse.ArgumentParser( + description='Scan git history for potentially leaked sensitive files', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python scripts/scan_git_history.py + python scripts/scan_git_history.py --verbose + python scripts/scan_git_history.py --output security_report.txt + python scripts/scan_git_history.py --verbose --output report.txt + """ + ) + parser.add_argument( + '--verbose', '-v', + action='store_true', + help='Enable verbose output' + ) + parser.add_argument( + '--output', '-o', + type=str, + help='Output file for report (default: print to stdout)' + ) + + args = parser.parse_args() + + # Check if we're in a git repository + try: + subprocess.run(['git', 'rev-parse', '--git-dir'], + capture_output=True, check=True) + except subprocess.CalledProcessError: + print("[ERROR] Not a git repository. Please run from repository root.") + sys.exit(1) + + # Run scan + scanner = GitHistoryScanner(verbose=args.verbose) + scanner.scan_repository() + + # Generate report + report = scanner.generate_report() + + # Output report + if args.output: + output_path = Path(args.output) + output_path.write_text(report) + print(f"\n[*] Report saved to: {output_path}") + print(f"[*] View with: cat {output_path}") + else: + print("\n" + report) + + # Exit with appropriate code + if scanner.findings: + print("\nāš  Potential security issues found. Please review the report.") + sys.exit(0) # Non-zero would break CI, use 0 and rely on report + else: + print("\nāœ“ No security issues detected.") + sys.exit(0) + + +if __name__ == '__main__': + main()