Mini-Project-2/mp2-2.py at main · AhmadAmali/Mini-Project-2 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# mini project 2 Phase 1, CMPUT291
import re
from functools import partial

import pymongo
import json
import time


def createCollections(db):
    # check if collections exist and drop if they do
    colList = db.list_collection_names()
    if "Posts" in colList:
        collection = db["Posts"]
        collection.drop()
    if "Tags" in colList:
        collection = db["Tags"]
        collection.drop()
    if "Votes" in colList:
        collection = db["Votes"]
        collection.drop()
    # create new collections
    posts = db["Posts"]
    tags = db["Tags"]
    votes = db["Votes"]

    # read json files and insert them into collections
    start = time.time()
    with open('Posts.json') as file:
        read_data = file.read()
        data = json.loads(read_data)
        posts.insert_many(data['posts']['row'])
        end = time.time()
        print("Posts collection created in ", end - start, "seconds")

    with open('Tags.json') as file:
        read_data = file.read()
        data = json.loads(read_data)
        tags.insert_many(data['tags']['row'])
        end = time.time()
        print("Tags collection created in ", end - start, "seconds")

    with open('Votes.json') as file:
        read_data = file.read()
        data = json.loads(read_data)
        votes.insert_many(data['votes']['row'])
        end = time.time()
        print("Votes collection created in ", end - start, "seconds")

        colList = db.list_collection_names()
        if "tags" in colList:
            print("Tags collection created successfully.\n")
        if "posts" in colList:
            print("Posts collection created successfully.\n")
        if "votes" in colList:
            print("Votes collection created successfully.\n")
        createTerms(db)


def createTerms(db):
    start = time.time()

    p1 = '[^a-zA-Z ]'
    p2 = '<.*?>'
    p3 = r'\W*\b\w{1,2}\b'
    pattern = re.compile("(%s|%s|%s)" % (p3, p1, p2))

    for doc in db.Posts.find().sort([("$natural", 1)]):
        raw_terms = ''
        try:
            raw_terms = doc['Title']
        except KeyError:
            # print("no title")
            pass
        try:
            raw_terms = raw_terms + '  ' + doc['Body']
        except KeyError:
            # print("no body")
            pass
        try:
            raw_terms = raw_terms + ' ' + doc["Tags"]
        except KeyError:
            pass
        try:
            db.Posts.update({"_id": doc['_id']}, {"$set": {"Terms": list(set(pattern.sub(' ', raw_terms.lower()).split()))}})
        except:
            pass
    end = time.time()
    #print("Terms Created in ", end - start, "seconds")


def main():
    port = int(input("Please enter the port you'd like to run the database on: "))
    client = pymongo.MongoClient("localhost", port)
    db = client['291db']
    createCollections(db)


if __name__ == "__main__":
    main()