diff --git a/.gitignore b/.gitignore index 451ef83..2257131 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ tutorial-env __pycache__ test.sublime-workspace -clearDatabase.py -testFaker.py \ No newline at end of file +clearDatabase.py \ No newline at end of file diff --git a/config.py b/config.py index d103a70..ca4830d 100644 --- a/config.py +++ b/config.py @@ -1,18 +1,24 @@ from faker import Faker +import random +import numpy as np +from bisect import bisect +from itertools import accumulate + fake = Faker() siteConfig = { - 'siteCount': 10 + 'siteCount': 20 } dbConfig = { - 'user' : 'root', - 'host' : 'localhost', - 'db' : 'test', - 'charset': 'latin1' + 'user' : 'root', + 'host' : 'localhost', + 'db' : 'bench', + 'password' : '12345', + 'charset' : 'latin1' } postsPopulatorConfig = { - 'postsCount': 60000, + 'postsCount': 1000000, 'batchSize' : 5000 } @@ -25,16 +31,12 @@ 'batchSize': 5000 } -rankGeneratorConfig = { - 'batchSize': 5000 -} - fakeText = "" def choice(weighted_choices): choices, weights = zip(*weighted_choices) cumdist = list(accumulate(weights)) - x = random() * cumdist[-1] + x = random.random() * cumdist[-1] weightedChoice = choices[bisect(cumdist, x)] return(weightedChoice) @@ -81,13 +83,16 @@ def countryChoice(result): result = countryChoice(result) return result +def randomRank(): + rankConfig = [] + for x in range(1, (postsPopulatorConfig['postsCount']) + 1): + rankConfig.append(x) + np.random.shuffle(rankConfig) + return rankConfig + def process(): global fakeText postsPopulatorConfig['postsCount'] //= siteConfig['siteCount'] fakeText = fake.text(max_nb_chars = 1000000) -from bisect import bisect -from itertools import accumulate -from random import random - process() \ No newline at end of file diff --git a/createDatabase.py b/createDatabase.py index a13e4e6..cef0288 100644 --- a/createDatabase.py +++ b/createDatabase.py @@ -17,8 +17,10 @@ def process(): connection = pymysql.connect(**dbConfig) except Exception as err: print("Connection error : {}".format(err)) - connection = pymysql.connect(user = dbConfig['user'], host = dbConfig['host'], charset = dbConfig['charset']) + connection = pymysql.connect(user = dbConfig['user'], host = dbConfig['host'], password = dbConfig['password'], charset = dbConfig['charset']) cursor = connection.cursor() createDatabase(cursor) + cursor.close() else: - print("DATABASE `{}` exists".format(dbConfig['db'])) \ No newline at end of file + print("DATABASE `{}` exists".format(dbConfig['db'])) + connection.close() \ No newline at end of file diff --git a/createTables.py b/createTables.py index b035619..7b9d312 100644 --- a/createTables.py +++ b/createTables.py @@ -9,14 +9,14 @@ def createTables(): " `site` varchar(20) NOT NULL," " `text` longtext NOT NULL," " `published` timestamp NOT NULL," - # " `published` int(11) NOT NULL," " `ES` tinyint(1) NOT NULL," " `US` tinyint(1) NOT NULL," " `MX` tinyint(1) NOT NULL," " `CO` tinyint(1) NOT NULL," - " `type` varchar(100) NOT NULL," + " `type` enum('normal', 'ecommerce', 'slideshow', 'video', 'duplicate', 'branded_club', 'brand_article', 'brand_article_video', 'longform', 'reposted_slideshow') NOT NULL DEFAULT 'normal'," " `url` varchar(255) NOT NULL," " `special` tinyint(1) NOT NULL," + " `rank` int(11) NOT NULL," " PRIMARY KEY (`id`)" ") ENGINE = InnoDB DEFAULT CHARSET=latin1") TABLES['post2tag'] = ( @@ -31,6 +31,48 @@ def createTables(): " `name` varchar(255) NOT NULL," " PRIMARY KEY (`id`)" ") ENGINE = InnoDB DEFAULT CHARSET=latin1") + TABLES['text'] = ( + "CREATE TABLE `text` (" + " `id` int(11) NOT NULL AUTO_INCREMENT," + " `p_text` longtext NOT NULL," + " PRIMARY KEY (`id`)" + ") ENGINE = InnoDB DEFAULT CHARSET=latin1") + TABLES['tag_query1'] = ( + "CREATE TABLE `tag_query1` (" + " `text_id` int(11) NOT NULL AUTO_INCREMENT," + " `p_site` varchar(20) NOT NULL," + " `p_ES` tinyint(1) NOT NULL," + " `p_US` tinyint(1) NOT NULL," + " `p_MX` tinyint(1) NOT NULL," + " `p_CO` tinyint(1) NOT NULL," + " `p_published` timestamp NOT NULL," + " PRIMARY KEY (`id`)" + ") ENGINE = InnoDB DEFAULT CHARSET=latin1") + TABLES['tag_query2'] = ( + "CREATE TABLE `tag_query2` (" + " `text_id` int(11) NOT NULL AUTO_INCREMENT," + " `p_site` varchar(20) NOT NULL," + " `p_ES` tinyint(1) NOT NULL," + " `p_US` tinyint(1) NOT NULL," + " `p_MX` tinyint(1) NOT NULL," + " `p_CO` tinyint(1) NOT NULL," + " `p_rank` int(11) NOT NULL," + " PRIMARY KEY (`id`)" + ") ENGINE = InnoDB DEFAULT CHARSET=latin1") + TABLES['tag_query3'] = ( + "CREATE TABLE `tag_query3` (" + " `text_id` int(11) NOT NULL AUTO_INCREMENT," + " `p_site` varchar(20) NOT NULL," + " `p_ES` tinyint(1) NOT NULL," + " `p_US` tinyint(1) NOT NULL," + " `p_MX` tinyint(1) NOT NULL," + " `p_CO` tinyint(1) NOT NULL," + " `p_rank` int(11) NOT NULL," + " `p_published` timestamp NOT NULL," + " `p_type` enum('normal', 'ecommerce', 'slideshow', 'video', 'duplicate', 'branded_club', 'brand_article', 'brand_article_video', 'longform', 'reposted_slideshow') NOT NULL DEFAULT 'normal'," + " PRIMARY KEY (`id`)" + ") ENGINE = InnoDB DEFAULT CHARSET=latin1") + for name, dbq in TABLES.items(): try: print("Creating table {}: ".format(name), end = '') diff --git a/main.py b/main.py index 8139688..3689c8b 100644 --- a/main.py +++ b/main.py @@ -3,7 +3,7 @@ from tagsPopulator import process as tagsPopulator from postsPopulator import process as postsPopulator from tagsRelationGenerator import process as tagsRelationGenerator -from rankGenerator import process as rankGenerator +from tagQueryPopulator import process as tagQueryPopulator from config import siteConfig from faker import Faker from timer import Timer, displayTimer @@ -24,6 +24,8 @@ def process(): displayTimer(timer.get_time_hhmmss(), 'Elapsed') tagsRelationGenerator() displayTimer(timer.get_time_hhmmss(), 'Elapsed') + tagQueryPopulator() + displayTimer(timer.get_time_hhmmss(), 'Elapsed') print("=====================================================") print("Mock Database Created") print("=====================================================") diff --git a/postsPopulator.py b/postsPopulator.py index 48ed106..60a55d8 100644 --- a/postsPopulator.py +++ b/postsPopulator.py @@ -1,8 +1,7 @@ import pymysql.cursors from random import randint from faker import Faker -from config import dbConfig, postTypeChoice, postsPopulatorConfig, fakeText, countryChoice, specialChoice -# from timer import Timestamp +from config import dbConfig, postTypeChoice, postsPopulatorConfig, fakeText, countryChoice, specialChoice, randomRank def batchInsertPosts(posts): query = ",".join(posts) @@ -17,19 +16,21 @@ def batchInsertPosts(posts): " `CO`," " `type`," " `url`," - " `special`" + " `special`," + " `rank`" ") VALUES " + query) cursor.execute(query) connection.commit() def insertPosts(): + rankCount = 0 posts = [] countriesWeighted = 0 + rankConfig = randomRank() for numberOfPosts in range(1, (postsPopulatorConfig['postsCount'] + 1)): - fakeTextStart = randint(0,997000) - fakeTextEnd = fakeTextStart + 3000 + fakeTextStart = randint(0,998000) + fakeTextEnd = fakeTextStart + 2000 text = fakeText[fakeTextStart:fakeTextEnd] + fake.pystr(max_chars = 20) - # published = timestamp.random() published = fake.date_time_between(start_date = "-6y", end_date = "now") countriesWeighted = countryChoice(countriesWeighted) ES = countriesWeighted[0] @@ -39,7 +40,9 @@ def insertPosts(): postType = postTypeChoice() url = fake.uri() + fake.pystr(max_chars = 10) special = specialChoice() - posts.append("('{}', '{}', '{}', {}, {}, {}, {}, '{}', '{}', {})" + rank = rankConfig[rankCount] + rankCount += 1 + posts.append("('{}', '{}', '{}', {}, {}, {}, {}, '{}', '{}', {}, {})" .format( site, text, @@ -50,7 +53,8 @@ def insertPosts(): CO, postType, url, - special)) + special, + rank)) if (numberOfPosts % postsPopulatorConfig['batchSize']) == 0: batchInsertPosts(posts) posts = [] @@ -61,12 +65,11 @@ def insertPosts(): print("Posts Insertion completed for {} posts".format(numberOfPosts)) def process(postSite): - global connection, cursor, fake#, timestamp + global connection, cursor, fake global site connection = pymysql.connect(**dbConfig) cursor = connection.cursor() fake = Faker() - # timestamp = Timestamp() site = postSite print("=====================================================") print("Generating Post data for ---> {}".format(site)) diff --git a/tagQueryPopulator.py b/tagQueryPopulator.py new file mode 100644 index 0000000..93b330e --- /dev/null +++ b/tagQueryPopulator.py @@ -0,0 +1,31 @@ +import pymysql.cursors +import config + +def tagQueryPopulate(): + query = "INSERT tag_query1 (p_site, p_ES, p_US, p_MX, p_CO, p_published) SELECT site, ES, US, MX, CO, published FROM wp_posts ORDER BY id ASC" + cursor.execute(query) + connection.commit() + print("Table 1 Populated") + query = "INSERT tag_query2 (p_site, p_ES, p_US, p_MX, p_CO, p_rank) SELECT site, ES, US, MX, CO, rank FROM wp_posts ORDER BY id ASC" + cursor.execute(query) + connection.commit() + print("Table 2 Populated") + query = "INSERT tag_query3 (p_site, p_ES, p_US, p_MX, p_CO, p_rank, p_published, p_type) SELECT site, ES, US, MX, CO, rank, published, type FROM wp_posts ORDER BY id ASC" + cursor.execute(query) + connection.commit() + print("Table 3 Populated") + query = "INSERT text (p_text) SELECT text FROM wp_posts ORDER BY id ASC" + cursor.execute(query) + connection.commit() + print("Table 4 Populated") + +def process(): + global connection, cursor + connection = pymysql.connect(**config.dbConfig) + cursor = connection.cursor() + print("=====================================================") + print("Populating Tag Query Tables") + tagQueryPopulate() + print("=====================================================") + cursor.close() + connection.close() \ No newline at end of file