From 2b50c04fd446495b67cdf906b661cd9582fe8f23 Mon Sep 17 00:00:00 2001 From: drtychai Date: Tue, 17 Nov 2020 09:50:00 -0800 Subject: [PATCH 1/3] recon-ng v5 fork --- Dockerfile | 26 ++++++ README.md | 67 +++++++++------ config_sample.py | 2 - enumall.py | 210 ++++++++++++++++++++++++----------------------- setup_enumall.sh | 44 ---------- 5 files changed, 178 insertions(+), 171 deletions(-) create mode 100644 Dockerfile delete mode 100644 config_sample.py delete mode 100755 setup_enumall.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..01f11a3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:20.04 +ENV HOME / +ENV TOOL_OUT /out +RUN apt update --fix-missing \ + && apt -y install git python3-dev python3-pip \ + && apt clean + +ENV LC_ALL "C.UTF-8" +ENV LANG "C.UTF-8" + +WORKDIR ${HOME} +RUN python3 -m pip install --upgrade pip setuptools wheel +RUN git clone https://github.com/lanmaster53/recon-ng \ + && ln -s /recon-ng/recon /recon \ + && ln -s /recon-ng/VERSION /VERSION +RUN git clone https://github.com/infosec-au/altdns + +RUN python3 -m pip install -r /recon-ng/REQUIREMENTS +RUN python3 -m pip install /altdns/ + +ADD words.txt /words.txt +ADD enumall.py /enumall.py + +RUN chmod +x /enumall.py +RUN mkdir -p /${TOOL_OUT} && chmod -R 700 /${TOOL_OUT} +ENTRYPOINT ["/enumall.py"] diff --git a/README.md b/README.md index 948ac1f..50b4cff 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Recon-ng and Alt-DNS are awesome. This script combines the power of these tools TLDR; I just want to do my subdomain discovery via ONE command and be done with it. -Only 1 module needs an api key (/api/google_site) find instructions for that on the recon-ng wiki. +Only 2 module needs api keys (`/api/google_site`, `/api/shodan`); find instructions for that on the recon-ng wiki. Script to enumerate subdomains, leveraging recon-ng. Uses google scraping, bing scraping, baidu scraping, yahoo scraping, netcraft, and bruteforces to find subdomains. Plus resolves to IP. @@ -14,41 +14,60 @@ Installation recon-ng from Source 1. Clone the Recon-ng repository - `git clone https://LaNMaSteR53@bitbucket.org/LaNMaSteR53/recon-ng.git` -2. Change into the Recon-ng directory. + `git clone https://github.com/lanmaster53/recon-ng` - `cd recon-ng` - -3. Install dependencies. +1. Change into the Recon-ng directory. - `pip install -r REQUIREMENTS` + `cd recon-ng` -4. Eventually link the installation directory to /usr/share/recon-ng +1. Install dependencies in a virtual environment: - `ln -s /$recon-ng_path /usr/share/recon-ng` + ```python + python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install venv + python3 -m venv .venv + + source .venv/bin/activate + python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install -r REQUIREMENTS + ``` -5. Optionally (highly recommended) download: +1. Symlink the `recon` lib and recon-ng `VERSION` from our clone to this repository: - + Alt-DNS (https://github.com/infosec-au/altdns) - + and a good subdomain bruteforce list (https://github.com/danielmiessler/SecLists/blob/master/Discovery/DNS/sorted_knock_dnsrecon_fierce_recon-ng.txt) + ```python + ln -s /path/to/recon-ng/recon ./recon + ln -s /path/to/recon-ng/VERSION ./VERSION + ``` -6. Create config.py file and specify the path to recon-ng and allDNS as it showed in config_sample.py +1. Optionally (highly recommended) download: + - [Alt-DNS][alt-dns] (`git clone https://github.com/infosec-au/altdns && python3 -m pip install altdns/`) + - and a good subdomain [bruteforce list][dns-wl] (`git clone https://github.com/danielmiessler/SecLists`) -# Basic Usage -`./enumall.py domain.com` +[alt-dns]: https://github.com/infosec-au/altdns +[dns-wl]: https://github.com/danielmiessler/SecLists/blob/master/Discovery/DNS/sorted_knock_dnsrecon_fierce_recon-ng.txt -also supports: -+ -w to run a custom wordlist with recon-ng -+ -a to use alt-dns -+ -p to feed a custom permutations list to alt-dns (requires -a flag) -+ -i to feed a list of domains (can also type extra domains into the original command) +# Usage -# Advanced Usage +``` +(.venv) ➜ ./ng-enum.py -h +usage: ng-enum.py [-h] [-a] [-i IN_FILE] [-o OUT_FILE] [-w WORDLIST] [-p PERMLIST] [domains ...] -`./enumall.py domain1.com domain2.com domain3.com -i domainlist.txt -a -p permutationslist.txt -w wordlist.com` +positional arguments: + domains one or more domains -Output from recon-ng will be in `.lst` and `.csv` files, output from alt-dns will be in a `.txt` file +optional arguments: + -h, --help show this help message and exit + -a After recon, run AltDNS? (this requires alt-dns) + -i IN_FILE input file of domains (one per line) + -o OUT_FILE output file for recon-ng results. if none specified, results not exported. + -w WORDLIST wordlist file for subdomain brute forcing. if none specified defaults to $RECON_HOME/data/hostnames.txt + -p PERMLIST input file of permutations for alt-dns. if none specified will use default list. +``` -by @jhaddix and @leifdreizler +## Docker +``` +docker build . -t domain:enumall +docker run -v ${PWD}:/out domain:enumall [-h] +``` diff --git a/config_sample.py b/config_sample.py deleted file mode 100644 index 016b7ee..0000000 --- a/config_sample.py +++ /dev/null @@ -1,2 +0,0 @@ -reconPath = "/usr/share/recon-ng/" -altDnsPath = "/root/Desktop/altdns-master/" diff --git a/enumall.py b/enumall.py index 15d0e55..14b51b6 100755 --- a/enumall.py +++ b/enumall.py @@ -1,108 +1,116 @@ -#!/usr/bin/env python - -# enumall is a refactor of enumall.sh providing a script to identify subdomains using several techniques and tools. -# Relying heavily on the stellar Recon-NG framework and Alt-DNS, enumall will identify subdomains via search engine -# scraping (yahoo, google, bing, baidu), identify subdomains using common OSINT sites (shodan, netcraft), identify -# concatenated subdomains (altDNS), and brute-forces with a stellar subdomain list (formed from Bitquark's subdomain -# research, Seclists, Knock, Fierce, Recon-NG, and more) located here: -# https://github.com/danielmiessler/SecLists/blob/master/Discovery/DNS/sorted_knock_dnsrecon_fierce_recon-ng.txt -# -# Alt-DNS Download: https://github.com/infosec-au/altdns -# -# by @jhaddix and @leifdreizler - +#!/usr/bin/env python3 +# enumall +# Authored by: @jhaddix and @leifdreizler +# Updated 2020-11: @bynx +from recon.core import base +from multiprocessing import Pool import argparse -import re -import sys import datetime -import time import os import sys -try: - from config import * -except: - reconPath = "/usr/share/recon-ng/" - altDnsPath = "/root/Desktop/altdns-master/" - -sys.path.insert(0,reconPath) -from recon.core import base -from recon.core.framework import Colors - -if altDnsPath: - sys.path.insert(1, altDnsPath) +def run_altdns(domains): + """Run altDNS with the given args.""" + for domain in domains: + altCmd = f"altdns" + subdomains = f"/output/subdomain/recon-ng-{domain}.out" + permList = f"/altdns/words.txt" + output = f"/output/subdomain/altdns-{domain}.out" + + # python altdns.py -i subdomainsList -o data_output -w permutationsList -r -s results_output.txt + print("running alt-dns... please be patient :) results will be displayed in "+output) + os.system(f"{altCmd} -i {subdomains} -o data_output -w {permList} -r -s {output}") + return + +def install_modules(reconBase, modules): + """Install required modules via recon-ng marketplace.""" + for module in modules: + reconBase._do_marketplace_install(module) + return def run_module(reconBase, module, domain): - x = reconBase.do_load(module) - x.do_set("SOURCE " + domain) - x.do_run(None) - - -def run_recon(domains, bruteforce): - stamp = datetime.datetime.now().strftime('%M:%H-%m_%d_%Y') - wspace = domains[0]+stamp - - reconb = base.Recon(base.Mode.CLI) - reconb.init_workspace(wspace) - reconb.onecmd("TIMEOUT=100") - module_list = ["recon/domains-hosts/bing_domain_web", "recon/domains-hosts/google_site_web", "recon/domains-hosts/netcraft", "recon/domains-hosts/shodan_hostname", "recon/netblocks-companies/whois_orgs", "recon/hosts-hosts/resolve"] - - for domain in domains: - for module in module_list: - run_module(reconb, module, domain) - - #subdomain bruteforcing - x = reconb.do_load("recon/domains-hosts/brute_hosts") - if bruteforce: - x.do_set("WORDLIST " + bruteforce) - else: - x.do_set("WORDLIST /usr/share/recon-ng/data/hostnames.txt") - x.do_set("SOURCE " + domain) - x.do_run(None) - - #reporting output - outFile = "FILENAME "+os.getcwd()+"/"+domains[0] - x = reconb.do_load("reporting/csv") - x.do_set(outFile+".csv") - x.do_run(None) - - x = reconb.do_load("reporting/list") - x.do_set(outFile+".lst") - x.do_set("COLUMN host") - x.do_run(None) - -parser = argparse.ArgumentParser() -parser.add_argument('-a', dest='runAltDns', action='store_true', help="After recon, run AltDNS? (this requires alt-dns)") -parser.add_argument("-i", dest="filename", type=argparse.FileType('r'), help="input file of domains (one per line)", default=None) -parser.add_argument("domains", help="one or more domains", nargs="*", default=None) -parser.add_argument("-w", dest="wordlist", type=argparse.FileType('r'), help="input file of subdomain wordlist. must be in same directory as this file, or give full path", default=None) -parser.add_argument("-p", dest="permlist", type=argparse.FileType('r'), help="input file of permutations for alt-dns. if none specified will use default list.", default=None) -args = parser.parse_args() - -if args.runAltDns and not altDnsPath: - print "Error: no altDns path specified, please download from: https://github.com/infosec-au/altdns" - exit(0) - -domainList = [] - -if args.domains: - domainList+=args.domains - -if args.filename: - lines = args.filename.readlines() - lines = [line.rstrip('\n') for line in lines] - domainList+=lines - -bruteforceList = args.wordlist.name if args.wordlist else "" - -run_recon(domainList, bruteforceList) - -if args.runAltDns: - workspace = domainList[0] - altCmd="python "+os.path.join(altDnsPath,"altdns.py") - subdomains = os.path.join(os.getcwd(), workspace+".lst") - permList = args.permlist.name if args.permlist else os.path.join(altDnsPath,"words.txt") - output = os.path.join(os.getcwd(),workspace+"_output.txt") - print "running alt-dns... please be patient :) results will be displayed in "+output - # python altdns.py -i subdomainsList -o data_output -w permutationsList -r -s results_output.txt - os.system('%s -i %s -o data_output -w %s -r -s %s' % (altCmd, subdomains, permList,output)) + """Run the passed module with options set.""" + try: + m = reconBase._do_modules_load(module) + m.options['source'] = domain + m.do_run(None) + except Exception as e: + print(f"[-] Exception hit: {e}") + raise + return + +def run_recon(domains, bf_wordlist, is_altdns_set, out_file): + """Initialize recon-ng base class and run core of script.""" + stamp = datetime.datetime.now().strftime('%M:%H-%m_%d_%Y') + wspace = domains[0]+stamp + + reconb = base.Recon(base.Mode.CLI) + reconb.start(base.Mode.CLI) + reconb._init_workspace(wspace) + + report_module = "reporting/list" + bf_module = "recon/domains-hosts/brute_hosts" + module_list = ["recon/hosts-hosts/resolve", "recon/domains-hosts/bing_domain_web", "recon/domains-hosts/google_site_web", + "recon/domains-hosts/shodan_hostname", "recon/netblocks-companies/whois_orgs", "recon/domains-hosts/netcraft"] + install_modules(reconb, module_list + [f"{bf_module}",f"{report_module}"]) + + pool = Pool() + procs = [] + for domain in domains: + for module in module_list: + p = pool.apply_async(run_module, args=(reconb, module, domain)) + procs.append(p) + + # subdomain bruteforcing if wordlist set + m = reconb._do_modules_load(bf_module) + m.options['wordlist'] = bf_wordlist + m.options['source'] = domain + m.do_run(None) + + # Export results if output file given + if out_file: + m = reconb._do_modules_load(report_module) + m.options['filename'] = out_file + m.options['column'] = "host" + m.do_run(None) + + if is_altdns_set: + run_altdns(domains) + return + +def main(argv): + domains = argv.domains + if argv.in_file: + try: + with argv.in_file as f: + domains += f.read() + except Exception as e: + print(f"[-] Exception hit: {e}") + + if not domains: + print("[-] No domain passed. Exiting...") + sys.exit(1) + run_recon(domains, argv.wordlist, argv.runAltDns, argv.out_file) + return + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('-a', dest='runAltDns', action='store_true', + help="After recon, run AltDNS? (this requires alt-dns)") + + parser.add_argument("-i", dest="in_file", type=argparse.FileType('r'), + help="input file of domains (one per line)", default=None) + + parser.add_argument("-o", dest="out_file", type=str, + help="output file for recon-ng results. if none specified, results not exported.", default=None) + + parser.add_argument("domains", help="one or more domains", nargs="*", default=None) + + parser.add_argument("-w", dest="wordlist", type=str, + help="wordlist file for subdomain brute forcing. if none specified defaults to $RECON_HOME/data/hostnames.txt", + default="words.txt") + + parser.add_argument("-p", dest="permlist", type=argparse.FileType('r'), + help="input file of permutations for alt-dns. if none specified will use default list.", default=None) + + main(parser.parse_args()) diff --git a/setup_enumall.sh b/setup_enumall.sh deleted file mode 100755 index 55ec941..0000000 --- a/setup_enumall.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# -# helps to setup domain, altdns, recon-ng tools -# Author: coreb1t - -VIRTENV=enumall -HOWTOFILE=how_to_use.txt - -echo "please enter the absolute path to the directory where the enumall.py tool should be installed" -echo " example: //tools/enumall" -read path - -if [ ! -d $path ];then - mkdir $path - echo "[+] directory $path created" -else - echo "[-] directory $path already exists" - echo "[-] exit" - exit -fi - -cd $path - -echo -e "[+] cloning git repos\n" -git clone https://LaNMaSteR53@bitbucket.org/LaNMaSteR53/recon-ng.git -git clone https://github.com/infosec-au/altdns.git -git clone https://github.com/jhaddix/domain.git -wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/sorted_knock_dnsrecon_fierce_recon-ng.txt - -cd domain -pathSed=$(echo $path | sed s/'\/'/'\\\/'/g) -sed -i "s/^reconPath.*/reconPath = \"$pathSed\/recon-ng\/\"/g" enumall.py -sed -i "s/^altDnsPath.*/altDnsPath = \"$pathSed\/altdns\/\"/g" enumall.py - -chmod 755 enumall.py - -# write how-to file -echo -e "\nIf you are using python virtualenv, excute workon $VIRTENV before running the script\n" > $HOWTOFILE -echo "./enumall.py -a -p ../altdns/words.txt -w ../sorted_knock_dnsrecon_fierce_recon-ng.txt" >> $HOWTOFILE - -# install virtualenv -echo -e "\n[+] configure the virtual env" -echo "[+] execute the following command" -echo " cd $path; mkvirtualenv $VIRTENV; pip install -r recon-ng/REQUIREMENTS; pip install -r altdns/requirements.txt" \ No newline at end of file From abbdd803681aeed8c24367e1b7e56e802aba900b Mon Sep 17 00:00:00 2001 From: drtychai Date: Tue, 17 Nov 2020 10:02:54 -0800 Subject: [PATCH 2/3] alt-dns tmp file --- .gitignore | 14 ++-- enumall.py | 22 ++--- words.txt | 232 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 18 deletions(-) create mode 100644 words.txt diff --git a/.gitignore b/.gitignore index 1164fd8..f1c64bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,6 @@ -.idea -config.py -virtualenv -.env -venv/ -ENV/ -*.csv -*.txt -*.lst +**/.venv +**/recon-ng +**/altdns +VERSION +recon data_output diff --git a/enumall.py b/enumall.py index 14b51b6..65da22b 100755 --- a/enumall.py +++ b/enumall.py @@ -11,15 +11,19 @@ def run_altdns(domains): """Run altDNS with the given args.""" - for domain in domains: - altCmd = f"altdns" - subdomains = f"/output/subdomain/recon-ng-{domain}.out" - permList = f"/altdns/words.txt" - output = f"/output/subdomain/altdns-{domain}.out" - - # python altdns.py -i subdomainsList -o data_output -w permutationsList -r -s results_output.txt - print("running alt-dns... please be patient :) results will be displayed in "+output) - os.system(f"{altCmd} -i {subdomains} -o data_output -w {permList} -r -s {output}") + + altCmd = "altdns" + subdomains = "altdns.in.tmp" + permList = "words.txt" + output = "altdns.out" + + with open(subdomains,"w") as f: + for domain in domains: + f.write(domain) + + print("[+] Running alt-dns...") + # python altdns.py -i subdomainsList -o data_output -w permutationsList -r -s results_output.txt + os.system(f"{altCmd} -i {subdomains} -o data_output -w {permList} -r -s {output}") return def install_modules(reconBase, modules): diff --git a/words.txt b/words.txt new file mode 100644 index 0000000..13c30e7 --- /dev/null +++ b/words.txt @@ -0,0 +1,232 @@ +1 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +2 +20 +2009 +2010 +2011 +2012 +2013 +2014 +2015 +2016 +2017 +2018 +2019 +3 +4 +5 +6 +7 +8 +9 +a +acc +accept +accounts +admin +admin1 +administrator +akali +akamai +alpha +alt +america +analytics +api +api1 +api-docs +apollo +april +aws +b +backend +beta +billing +boards +box +brand +brasil +brazil +bucket +bucky +c +cdn +cf +chef +ci +client +cloudfront +cms +cms1 +cn +com +confluence +container +control +data +dec +demo +dev +dev1 +developer +devops +docker +docs +drop +edge +elasticbeanstalk +elb +email +eng +engima +engine +engineering +eu +europe +europewest +euw +euwe +evelynn +events +feb +fet +firewall +forms +forum +frontpage +fw +games +germany +gh +ghcpi +git +github +global +hkg +hw +hwcdn +i +ids +int +internal +jenkins +jinx +july +june +kor +korea +kr +lan +las +latin +latinamerica +lax +lax1 +lb +loadbalancer +login +machine +mail +march +merch +mirror +na +nautilus +net +netherlands +nginx +nl +node +northamerica +nov +oceania +oct +ops +org +origin +page +pantheon +pass +pay +payment +pc +php +pl +poland +preferences +priv +private +prod +production +profile +profiles +promo +promotion +proxy +redirector +region +repo +repository +reset +restrict +restricted +reviews +s +s3 +sandbox +search +secure +security +sept +server +service +singed +skins +spring +ssl +staff +stage +stage1 +staging +static +support +swagger +system +t +train +training +team +test +test1 +testbed +testing +testing1 +tomcat +tpe +tr +trial +tur +turk +turkey +twitch +uat +v1 +v2 +vi +vpn +w3 +web +web1 +webapp +westeurope +z From 806f64fc8ae4b1f898a328f4268867b5c2fb8376 Mon Sep 17 00:00:00 2001 From: drtychai Date: Tue, 17 Nov 2020 10:19:37 -0800 Subject: [PATCH 3/3] usage menu typo fixed --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 50b4cff..af77d00 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,8 @@ Installation recon-ng from Source # Usage ``` -(.venv) ➜ ./ng-enum.py -h -usage: ng-enum.py [-h] [-a] [-i IN_FILE] [-o OUT_FILE] [-w WORDLIST] [-p PERMLIST] [domains ...] +(.venv) ➜ ./enumall.py -h +usage: enumall.py [-h] [-a] [-i IN_FILE] [-o OUT_FILE] [-w WORDLIST] [-p PERMLIST] [domains ...] positional arguments: domains one or more domains