gitgetinfo/gitgetinfo.py at master · andrewdmcleod/gitgetinfo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
#!/usr/bin/env python
from BeautifulSoup import BeautifulSoup
import json
import argparse
import urllib2
import sys

orgs = []
users = []
branch = []
filename = []

raw_github = "https://raw.githubusercontent.com"
github = "http://github.com"

parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, \
                                 description='Search juju-solutions repos for layers and interfaces',
                                 epilog='org, user and branch are all appended arguments, e.g.: \n' \
                                 'gitgetinfo.py -s basic -b master -b featurebranch1 -b featurebranch2' \
                                 ' -u bobsmith -u mikepeters -o org1 -o org2')
parser.add_argument("-s, --search", required=True, action="store", help="layer or interface name (or substring)", dest="search")
parser.add_argument("-b, --branch", action="append", help="specify additional branch names (default is master)", dest="branch")
parser.add_argument("-o, --org", action="append", help="specify additional github org names to scan", dest="org")
parser.add_argument("-u, --user", action="append", help="specify additional github user repos to scan", dest="user")
parser.add_argument("-f, --filename", required=True, action="append", help="filename to scan in repo for search string", dest="filename")
args = parser.parse_args()

if args.filename:
    for name in args.filename:
        filename.append(name)

if args.org:
    for name in args.org:
        orgs.append(name)

if args.user:
    for name in args.user:
        users.append(name)

if args.branch:
    for name in args.branch:
        branch.append(name)
else:
    branch.append("master")

if not args.search:
    parser.print_help()
    sys.exit(0)
else:
    search = args.search

def scan_org(org, branch, file):
    print "\nScanning {} for {}[{}] in {}...\n".format(org, search, branch, file)
    full_url = 'https://github.com/' + org
    try:
        open_org = urllib2.urlopen(full_url)
    except urllib2.HTTPError as e:
        print "\nBad organisation name, {}\n".format(e)
        sys.exit(1)
    soup = BeautifulSoup(open_org)
    for link in soup.findAll('a'):
        if 'itemprop' in str(link) and 'layer' in str(link):
            scan_url = raw_github + str(link.get('href')) + '/' + branch + '/' + file
            git_url = github + str(link.get('href')) + '/blob/' + branch + '/' + file
            try:
                onions = urllib2.urlopen(scan_url)
                onion_soup = BeautifulSoup(onions)
                d = onion_soup.findAll(text=True)
                for line in d:
                    for line2 in line.strip().split('\n'):
                        if 'include' in line2:
                            if search in line2:
                                print search + " Found in: " + git_url
                                print line2 + "\n"
            except:
                pass

def scan_user(user, branch, filename):
    print "\nScanning {} for {}[{}] in {}...\n".format(user, search, branch, filename)
    full_url = 'https://api.github.com/users/' + user + '/repos?page=1&per_page=100'
    try:
        open_user = urllib2.urlopen(full_url)
    except urllib2.HTTPError as e:
        print "\nBad username, {}\n".format(e)
        sys.exit(1)
    user_data = json.load(open_user)
    for repo in user_data:
        scan_url = raw_github + '/' + repo['full_name'] + '/' + branch + '/' + filename
        git_url = github + '/' + repo['full_name'] + '/blob/' + branch + '/' + filename
        try:
            onions = urllib2.urlopen(scan_url)
            onion_soup = BeautifulSoup(onions)
            d = onion_soup.findAll(text=True)
            for line in d:
                for line2 in line.strip().split('\n'):
                    if search in str(line2):
                        print search + " Found in: " + git_url
                        print line2 + "\n"
        except:
            pass

if orgs:
    for org in orgs:
        for br in branch:
            for fn in filename:
                scan_org(org, br, fn)

if users:
    for user in users:
        for br in branch:
            for fn in filename:
                scan_user(user, br, fn)