From 5889a4c1a3e18e5b6e37c8c5333abfe635fa0837 Mon Sep 17 00:00:00 2001 From: Xnuvers007 Date: Fri, 15 May 2026 06:49:32 +0700 Subject: [PATCH 1/3] refactor: modularize core scanning engine and modernize Docker environment for improved detection and scalability --- .gitignore | 8 + BurpSuite-SecretFinder/README.md | 2 +- BurpSuite-SecretFinder/SecretFinder.py | 95 +-- Dockerfile | 58 +- README.md | 500 +++++++++++--- README_EN.md | 153 +++++ SecretFinder.py | 868 ++++++++++++------------- core/__init__.py | 16 + core/fetcher.py | 115 ++++ core/input_parser.py | 268 ++++++++ core/output.py | 439 +++++++++++++ core/patterns.py | 778 ++++++++++++++++++++++ core/scanner.py | 203 ++++++ images/banner.png | Bin 0 -> 927410 bytes images/before.png | Bin 0 -> 210304 bytes images/image.png | Bin 0 -> 88543 bytes requirements.txt | 9 +- 17 files changed, 2913 insertions(+), 599 deletions(-) create mode 100644 README_EN.md create mode 100644 core/__init__.py create mode 100644 core/fetcher.py create mode 100644 core/input_parser.py create mode 100644 core/output.py create mode 100644 core/patterns.py create mode 100644 core/scanner.py create mode 100644 images/banner.png create mode 100644 images/before.png create mode 100644 images/image.png diff --git a/.gitignore b/.gitignore index 2d19fc7..c59dc97 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,9 @@ *.html +venv/ +output/*.html +output/*.json +output/*.csv +output/*.txt +__pycache__/* +core/__pycache__/* +BurpSuite-SecretFinder/__pycache__/* diff --git a/BurpSuite-SecretFinder/README.md b/BurpSuite-SecretFinder/README.md index 9b97241..59b0a89 100644 --- a/BurpSuite-SecretFinder/README.md +++ b/BurpSuite-SecretFinder/README.md @@ -1,4 +1,4 @@ -## Burp Suite - Secret Finder (beta v0.1) +## Burp Suite - Secret Finder (v2.0) A Burp Suite extension to help pentesters to discover a apikeys,accesstokens and more sensitive data using a regular expressions. SecretFinder process any HTTP response (support javascript file) and support Passive and Active scan. This extension has been developed by (@m4ll0k). diff --git a/BurpSuite-SecretFinder/SecretFinder.py b/BurpSuite-SecretFinder/SecretFinder.py index 2cc994d..dd28c3f 100644 --- a/BurpSuite-SecretFinder/SecretFinder.py +++ b/BurpSuite-SecretFinder/SecretFinder.py @@ -34,44 +34,65 @@ def consolidateDuplicateIssues(self, existingIssue, newIssue): return 0 # add your regex here + # Expanded patterns from Advanced Edition regexs = { - 'google_api' : 'AIza[0-9A-Za-z-_]{35}', - 'docs_file_exetension' : '^.*\.(xls|xlsx|doc|docx)$', - 'bitcoin_address' : '([13][a-km-zA-HJ-NP-Z0-9]{26,33})', - 'slack_api_key' : 'xox.-[0-9]{12}-[0-9]{12}-[0-9a-zA-Z]{24}', - 'us_cn_zipcode' : '/(^\d{5}(-\d{4})?$)|(^[ABCEGHJKLMNPRSTVXY]{1}\d{1}[A-Z]{1} *\d{1}[A-Z]{1}\d{1}$)/', - 'google_cloud_platform_auth' : '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}', - 'google_cloud_platform_api' : '[A-Za-z0-9_]{21}--[A-Za-z0-9_]{8}', - 'amazon_secret_key' : '[0-9a-zA-Z/+]{40}', - 'gmail_auth_token' : '[0-9(+-[0-9A-Za-z_]{32}.apps.qooqleusercontent.com', - 'github_auth_token' : '[0-9a-fA-F]{40}', - 'Instagram_token' : '[0-9a-fA-F]{7}.[0-9a-fA-F]{32}', - 'twitter_access_token' : '[1-9][ 0-9]+-(0-9a-zA-Z]{40}', - 'firebase' : 'AAAA[A-Za-z0-9_-]{7}:[A-Za-z0-9_-]{140}', - 'google_captcha' : '6L[0-9A-Za-z-_]{38}|^6[0-9a-zA-Z_-]{39}$', - 'google_oauth' : 'ya29\.[0-9A-Za-z\-_]+', - 'amazon_aws_access_key_id' : 'A[SK]IA[0-9A-Z]{16}', - 'amazon_mws_auth_toke' : 'amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', - 'amazon_aws_url' : 's3\.amazonaws.com[/]+|[a-zA-Z0-9_-]*\.s3\.amazonaws.com', - 'facebook_access_token' : 'EAACEdEose0cBA[0-9A-Za-z]+', - 'authorization_basic' : 'basic\s*[a-zA-Z0-9=:_\+\/-]+', - 'authorization_bearer' : 'bearer\s*[a-zA-Z0-9_\-\.=:_\+\/]+', - 'authorization_api' : 'api[key|\s*]+[a-zA-Z0-9_\-]+', - 'mailgun_api_key' : 'key-[0-9a-zA-Z]{32}', - 'twilio_api_key' : 'SK[0-9a-fA-F]{32}', - 'twilio_account_sid' : 'AC[a-zA-Z0-9_\-]{32}', - 'twilio_app_sid' : 'AP[a-zA-Z0-9_\-]{32}', - 'paypal_braintree_access_token' : 'access_token\$production\$[0-9a-z]{16}\$[0-9a-f]{32}', - 'square_oauth_secret' : 'sq0csp-[ 0-9A-Za-z\-_]{43}|sq0[a-z]{3}-[0-9A-Za-z\-_]{22,43}', - 'square_access_token' : 'sqOatp-[0-9A-Za-z\-_]{22}|EAAA[a-zA-Z0-9]{60}', - 'stripe_standard_api' : 'sk_live_[0-9a-zA-Z]{24}', - 'stripe_restricted_api' : 'rk_live_[0-9a-zA-Z]{24}', - 'github_access_token' : '[a-zA-Z0-9_-]*:[a-zA-Z0-9_\-]+@github\.com*', - 'rsa_private_key' : '-----BEGIN RSA PRIVATE KEY-----', - 'ssh_dsa_private_key' : '-----BEGIN DSA PRIVATE KEY-----', - 'ssh_dc_private_key' : '-----BEGIN EC PRIVATE KEY-----', - 'pgp_private_block' : '-----BEGIN PGP PRIVATE KEY BLOCK-----', - 'json_web_token' : 'ey[A-Za-z0-9_-]*\.[A-Za-z0-9._-]*|ey[A-Za-z0-9_\/+-]*\.[A-Za-z0-9._\/+-]*' + 'google_api_key': 'AIza[0-9A-Za-z\-_]{35}', + 'google_oauth2': 'ya29\.[0-9A-Za-z\-_]+', + 'docs_file_extension': '(?i)\\.(xlsx|xlsm|xlsb|xls|csv|xml|mht|mhtml|html|htm|xltx|xltm|xlt|txt|prn|dif|slk|xlam|xla|pdf|xps|ods|docx|docm|doc|dotx|dotm|dot|rtf|odt)', + 'bitcoin_address': '\\b[13][a-km-zA-HJ-NP-Z0-9]{26,33}\\b', + 'google_cloud_platform_auth': '\\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\\b', + 'google_cloud_platform_api': '\\b[A-Za-z0-9_]{21}--[A-Za-z0-9_]{8}\\b', + 'instagram_token': '\\b[0-9a-fA-F]{7}\\.[0-9a-fA-F]{32}\\b', + 'gmail_auth_token': '[0-9]{12}-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com', + 'global_postal_code': '\\b\\d{5}(?:[-\\s]\\d{4})?\\b|\\b[A-Z]\\d[A-Z]\\s?\\d[A-Z]\\d\\b|\\b[A-Z]{1,2}\\d[A-Z\\d]?\\s?\\d[A-Z]{2}\\b|\\b\\d{3}-\\d{4}\\b|\\b\\d{4,6}\\b', + 'slack_api_key': 'xox.-[0-9]{12}-[0-9]{12}-[0-9a-zA-Z]{24}', + 'amazon_secret_key': '(?i)aws_secret_access_key\\s*=\\s*[A-Za-z0-9/+=]{40}', + 'github_auth_token': '[0-9a-fA-F]{40}', + 'twitter_access_token': '[1-9][ 0-9]+-(0-9a-zA-Z]{40}', + 'firebase': 'AAAA[A-Za-z0-9_-]{7}:[A-Za-z0-9_-]{140}', + 'google_captcha': '6L[0-9A-Za-z-_]{38}|^6[0-9a-zA-Z_-]{39}$', + 'google_oauth': 'ya29\\.[0-9A-Za-z\\-_]+', + 'aws_access_key_id': 'A[SK]IA[0-9A-Z]{16}', + 'aws_secret_access_key_v2': '(?i)aws_secret_access_key\\s*=\\s*[A-Za-z0-9/+=]{40}', + 'aws_mws_token': 'amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', + 'amazon_s3_url': '[a-zA-Z0-9\\-\\.\\_]+\\.s3\\.amazonaws\\.com|s3://[a-zA-Z0-9\\-\\.\\_]+|s3\\.amazonaws\\.com/[a-zA-Z0-9\\-\\.\\_]+', + 'azure_storage_key': 'DefaultEndpointsProtocol=https?;AccountName=[^;]+;AccountKey=[A-Za-z0-9+/=]{88}', + 'azure_sas_token': 'sv=\\d{4}-\\d{2}-\\d{2}&s[a-z]=&sig=[A-Za-z0-9%+/=]+', + 'facebook_access_token': 'EAACEdEose0cBA[0-9A-Za-z]+', + 'facebook_app_secret': '(?i)(facebook|fb)(.{0,20})([0-9a-f]{32})', + 'github_pat_classic': 'ghp_[A-Za-z0-9]{36}', + 'github_pat_fine': 'github_pat_[A-Za-z0-9_]{82}', + 'github_oauth': 'gho_[A-Za-z0-9]{36}', + 'github_cred_url': '[a-zA-Z0-9_\\-]*:[a-zA-Z0-9_\\-]+@github\\.com', + 'gitlab_token': 'glpat-[A-Za-z0-9\\-_]{20}', + 'stripe_live_key': 'sk_live_[0-9a-zA-Z]{24}', + 'stripe_restricted_key': 'rk_live_[0-9a-zA-Z]{24}', + 'slack_bot_token': 'xoxb-[0-9]{11}-[0-9]{11}-[a-zA-Z0-9]{24}', + 'slack_user_token': 'xoxp-[0-9]+-[0-9]+-[0-9]+-[a-f0-9]+', + 'slack_webhook': 'https://hooks\\.slack\\.com/services/T[a-zA-Z0-9_]+/B[a-zA-Z0-9_]+/[a-zA-Z0-9_]+', + 'twilio_api_key': 'SK[0-9a-fA-F]{32}', + 'twilio_account_sid': 'AC[a-zA-Z0-9_\\-]{32}', + 'paypal_braintree_token': 'access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}', + 'mailgun_api_key': 'key-[0-9a-zA-Z]{32}', + 'sendgrid_api_key': 'SG\\.[a-zA-Z0-9\\-_]{22}\\.[a-zA-Z0-9\\-_]{43}', + 'mailchimp_api_key': '[0-9a-f]{32}-us[0-9]{1,2}', + 'jwt_token': 'ey[A-Za-z0-9_\\-]{10,}\\.[A-Za-z0-9_\\-]{10,}\\.[A-Za-z0-9_\\-\\.+/=]{10,}', + 'basic_auth_header': '(?i)basic\\s+[a-zA-Z0-9=:_+/\\-]{10,200}', + 'bearer_token': '(?i)bearer\\s+[a-zA-Z0-9_\\-\\..=:+/]{10,500}', + 'db_connection_string': '(?i)(mongodb|mysql|postgres|postgresql|redis|mssql|oracle)\\://[^\\s\'"<>]+', + 'hardcoded_password': '(?i)(password|passwd|pwd|secret|credentials?)\\s*[`=:"\\[]+\\s*[^\\s,;\\]]{4,}', + 'discord_bot_token': '[MN][A-Za-z\\d]{23}\\.[\\w\\-]{6}\\.[\\w\\-]{27}', + 'telegram_bot_token': '[0-9]{9}:[a-zA-Z0-9_\\-]{35}', + 'shopify_token': 'shpat_[a-fA-F0-9]{32}|shpca_[a-fA-F0-9]{32}|shppa_[a-fA-F0-9]{32}', + 'openai_api_key': 'sk-[A-Za-z0-9]{48}|sk-proj-[A-Za-z0-9\\-_]{90,}', + 'anthropic_api_key': 'sk-ant-api\\d{2}-[A-Za-z0-9\\-_]{93}AA', + 'huggingface_token': 'hf_[A-Za-z0-9]{37}', + 'kubernetes_token': '(?i)(kube[_\\-\\s]?token|k8s[_\\-\\s]?token|KUBERNETES_TOKEN)\\s*[=:]\\s*["\']?ey[A-Za-z0-9_\\-]{50,}["\']?', + 'cloudflare_api_token': '(?i)(cloudflare[_\\-\\s]?(api[_\\-\\s]?)?token)\\s*[=:]\\s*["\']?[A-Za-z0-9_\\-]{40}["\']?', + 'ethereum_private_key': '(?i)(eth[_\\-\\s]?private[_\\-\\s]?key|PRIVATE_KEY)\\s*[=:]\\s*["\']?0x[a-fA-F0-9]{64}["\']?', + 'rsa_private_key': '-----BEGIN RSA PRIVATE KEY-----', + 'dsa_private_key': '-----BEGIN DSA PRIVATE KEY-----', + 'generic_private_key': '-----BEGIN [^\\s]+ PRIVATE KEY-----', } regex = r"[:|=|\'|\"|\s*|`|ยด| |,|?=|\]|\|//|/\*}](%%regex%%)[:|=|\'|\"|\s*|`|ยด| |,|?=|\]|\}|&|//|\*/]" issuename = "SecretFinder: %s" diff --git a/Dockerfile b/Dockerfile index 6ae09c4..f134381 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,44 @@ -# usage: -# while read url; do docker run -t wfnintr/secretfinder -i $url -o cli | tee -a js_results.txt;done < urls.txt -from python:alpine -LABEL source="SecretFinder " -LABEL maintainer="wfnintr@null.net" -RUN apk update && \ - apk add --virtual build-deps \ - build-base gcc python3-dev && \ - apk add libxml2-dev libxslt-dev && \ - wget https://raw.githubusercontent.com/m4ll0k/SecretFinder/master/requirements.txt -qO - | pip3 install -r /dev/stdin && \ - wget https://raw.githubusercontent.com/m4ll0k/SecretFinder/master/SecretFinder.py -qO /usr/local/bin/SecretFinder.py && \ - chmod +x /usr/local/bin/SecretFinder.py && \ - apk del build-deps -ENTRYPOINT [ "SecretFinder.py" ] +# SecretFinder Advanced Edition - Dockerized +# Build: docker build -t secretfinder . +# Run: docker run --rm -v $(pwd)/output:/app/output secretfinder -i https://example.com/app.js -o output/result.html + +FROM python:3.12-slim + +# Labels for metadata +LABEL org.opencontainers.image.title="SecretFinder Advanced" +LABEL org.opencontainers.image.description="Professional tool to discover secrets in JavaScript files" +LABEL org.opencontainers.image.authors="Xnuvers007 " + +# Environment variables +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + libxml2-dev \ + libxslt-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Create non-root user and setup directories +RUN useradd -m scanner && \ + mkdir -p /app/output && \ + chown -R scanner:scanner /app + +# Copy project files +COPY --chown=scanner:scanner . . + +# Switch to non-root user +USER scanner + +# Ensure output directory is a volume +VOLUME ["/app/output"] + +ENTRYPOINT ["python", "SecretFinder.py"] +CMD ["--help"] diff --git a/README.md b/README.md index 252a220..a0a973f 100644 --- a/README.md +++ b/README.md @@ -1,134 +1,452 @@ +# ๐Ÿ” SecretFinder โ€” Advanced Edition -## about SecretFinder +![SecretFinder Banner](images/banner.png) -SecretFinder is a python script based on [LinkFinder](https://github.com/GerbenJavado/LinkFinder), written to discover sensitive data like apikeys, accesstoken, authorizations, jwt,..etc in JavaScript files. It does so by using jsbeautifier for python in combination with a fairly large regular expression. The regular expressions consists of four small regular expressions. These are responsible for finding and search anything on js files. +> **Rewritten & Enhanced by [Xnuvers007](https://github.com/Xnuvers007/SecretFinder)** +> [English Version (README_EN.md)](README_EN.md) +> Original by [m4ll0k](https://github.com/m4ll0k/SecretFinder) ยท Based on [LinkFinder](https://github.com/GerbenJavado/LinkFinder) -The output is given in HTML or plaintext. +SecretFinder adalah **tool Python profesional** untuk menemukan data sensitif โ€” API keys, access token, JWT, credentials, private key, dan lainnya โ€” yang tersembunyi di dalam file JavaScript. -![main](https://i.imgur.com/D7MT2KL.png) +--- +## โœจ Fitur Advanced Edition +| Fitur | Original | Advanced Edition | +|---|---|---| +| Regex patterns | ~20 | **118+** | +| Severity scoring | โœ— | โœ… CRITICAL / HIGH / MEDIUM / LOW / INFO | +| Output format | HTML saja | **HTML dark ยท JSON ยท CSV ยท CLI berwarna** | +| Concurrent scan | โœ— | โœ… ThreadPoolExecutor (configurable) | +| Retry logic | โœ— | โœ… Exponential back-off | +| Output Folder | โœ— | โœ… Auto-creates `output/` folder | +| Auto-increment | โœ— | โœ… `results.html` โ†’ `results2.html` (no overwrite) | +| Input bare domain | โœ— | โœ… `example.com` โ†’ auto `https://example.com` | +| Input alias | `-i` | โœ… `-i` / `-u` / `--input` / `--url` | +| Scheme pilihan | โœ— | โœ… `--http` / `--https` (default: https) | +| Arsitektur | 1 file | โœ… Modular (`core/` package) | +| HTML report | Basic | โœ… **Premium dark-mode dashboard** | +| Burp Extension | Basic | โœ… **Updated with 118+ Patterns** | -## Help +--- +## ๐Ÿ“ฆ Instalasi + +```bash +git clone https://github.com/Xnuvers007/SecretFinder.git +cd SecretFinder + +# Buat virtual environment (opsional tapi disarankan) +python -m venv venv +venv\Scripts\activate # Windows +source venv/bin/activate # Linux/Mac + +pip install -r requirements.txt +python SecretFinder.py --help +``` + +--- + +## ๐Ÿš€ Panduan Penggunaan โ€” Dari A sampai Z + +### 1. Input paling simpel โ€” cukup ketik domain + +```bash +python SecretFinder.py -i example.com +python SecretFinder.py -i www.example.com +python SecretFinder.py -i sub.domain.example.com +python SecretFinder.py -i www.api.v2.staging.example.co.uk # multi-level subdomain +``` +> Semua otomatis jadi `https://...`. Tidak perlu ketik `http://` atau `https://`. + +--- + +### 1b. Input IPv4 (dengan/tanpa port) + +```bash +python SecretFinder.py -i 192.168.1.1 -o cli +python SecretFinder.py -i 192.168.1.1:8080 -o cli +python SecretFinder.py -i 10.0.0.1:9200/api --http -o cli # paksa http +python SecretFinder.py -i 172.16.0.1:3000/swagger -e +``` + +--- + +### 1c. Input IPv6 + +```bash +python SecretFinder.py -i "[::1]:8080" -o cli +python SecretFinder.py -i "[fe80::1]:443" --http -o cli +``` + +--- + +### 1d. Domain + port + path + +```bash +python SecretFinder.py -i example.com:8443 -e +python SecretFinder.py -i api.example.com:3000/v2 -o cli +python SecretFinder.py -i internal.corp.local:8080/graphql -o cli --http +``` + + +### 2. Scan seluruh JS di sebuah domain (mode extract) + +```bash +python SecretFinder.py -i example.com -e +``` +> `-e` / `--extract` โ†’ ambil semua `