denormalized query tables

.gitignore

@@ -1,5 +1,4 @@
 tutorial-env
 __pycache__
 test.sublime-workspace
-clearDatabase.py
-testFaker.py
+clearDatabase.py

config.py

@@ -1,18 +1,24 @@
 from faker import Faker
+import random
+import numpy as np
+from bisect import bisect
+from itertools import accumulate
+
 fake = Faker()
 siteConfig = {
-    'siteCount': 10
+    'siteCount': 20
 }
 
 dbConfig = {
-    'user'   : 'root',
-    'host'   : 'localhost',
-    'db'     : 'test',
-    'charset': 'latin1'
+    'user'      : 'root',
+    'host'      : 'localhost',
+    'db'        : 'bench',
+    'password'  : '12345',
+    'charset'   : 'latin1'
 }
 
 postsPopulatorConfig = {
-    'postsCount': 60000,
+    'postsCount': 1000000,
     'batchSize' : 5000
 }
 
@@ -25,16 +31,12 @@
     'batchSize': 5000
 }
 
-rankGeneratorConfig = {
-    'batchSize': 5000
-}
-
 fakeText = ""
 
 def choice(weighted_choices):
     choices, weights = zip(*weighted_choices)
     cumdist = list(accumulate(weights))
-    x = random() * cumdist[-1]
+    x = random.random() * cumdist[-1]
     weightedChoice = choices[bisect(cumdist, x)]
     return(weightedChoice)
 
@@ -81,13 +83,16 @@ def countryChoice(result):
         result = countryChoice(result) 
     return result
 
+def randomRank():
+    rankConfig = []
+    for x in range(1, (postsPopulatorConfig['postsCount']) + 1):
+        rankConfig.append(x)
+    np.random.shuffle(rankConfig)
+    return rankConfig
+
 def process():
     global fakeText
     postsPopulatorConfig['postsCount'] //= siteConfig['siteCount'] 
     fakeText = fake.text(max_nb_chars = 1000000)
 
-from bisect import bisect
-from itertools import accumulate
-from random import random
-
 process()

createDatabase.py

@@ -17,8 +17,10 @@ def process():
         connection = pymysql.connect(**dbConfig)
     except Exception as err:
         print("Connection error : {}".format(err))
-        connection = pymysql.connect(user = dbConfig['user'], host = dbConfig['host'], charset = dbConfig['charset'])
+        connection = pymysql.connect(user = dbConfig['user'], host = dbConfig['host'], password = dbConfig['password'], charset = dbConfig['charset'])
         cursor = connection.cursor()
         createDatabase(cursor)
+        cursor.close()
     else:
-        print("DATABASE `{}` exists".format(dbConfig['db']))
+        print("DATABASE `{}` exists".format(dbConfig['db']))
+    connection.close()

createTables.py

@@ -9,14 +9,14 @@ def createTables():
         "   `site`      varchar(20) NOT NULL,"
         "   `text`      longtext NOT NULL,"
         "   `published` timestamp NOT NULL,"
-        # "   `published` int(11) NOT NULL,"
         "   `ES`        tinyint(1) NOT NULL,"
         "   `US`        tinyint(1) NOT NULL,"
         "   `MX`        tinyint(1) NOT NULL,"
         "   `CO`        tinyint(1) NOT NULL,"
-        "   `type`      varchar(100) NOT NULL,"
+        "   `type`      enum('normal', 'ecommerce', 'slideshow', 'video', 'duplicate', 'branded_club', 'brand_article', 'brand_article_video', 'longform', 'reposted_slideshow') NOT NULL DEFAULT 'normal',"
         "   `url`       varchar(255) NOT NULL,"
         "   `special`   tinyint(1) NOT NULL,"
+        "   `rank`      int(11) NOT NULL,"
         "   PRIMARY KEY (`id`)"
         ")  ENGINE = InnoDB DEFAULT CHARSET=latin1")
     TABLES['post2tag'] = (
@@ -31,6 +31,48 @@ def createTables():
         "   `name`      varchar(255) NOT NULL,"
         "   PRIMARY KEY (`id`)"
         ")  ENGINE = InnoDB DEFAULT CHARSET=latin1")
+    TABLES['text'] = (
+        "CREATE TABLE   `text` ("
+        "   `id`        int(11) NOT NULL AUTO_INCREMENT,"
+        "   `p_text`    longtext NOT NULL,"
+        "   PRIMARY KEY (`id`)"
+        ")  ENGINE = InnoDB DEFAULT CHARSET=latin1")
+    TABLES['tag_query1'] = (
+        "CREATE TABLE   `tag_query1` ("
+        "   `text_id`   int(11) NOT NULL AUTO_INCREMENT,"
+        "   `p_site`      varchar(20) NOT NULL,"
+        "   `p_ES`        tinyint(1) NOT NULL,"
+        "   `p_US`        tinyint(1) NOT NULL,"
+        "   `p_MX`        tinyint(1) NOT NULL,"
+        "   `p_CO`        tinyint(1) NOT NULL,"
+        "   `p_published` timestamp NOT NULL,"
+        "   PRIMARY KEY (`id`)"
+        ")  ENGINE = InnoDB DEFAULT CHARSET=latin1")
+    TABLES['tag_query2'] = (
+        "CREATE TABLE   `tag_query2` ("
+        "   `text_id`     int(11) NOT NULL AUTO_INCREMENT,"
+        "   `p_site`      varchar(20) NOT NULL,"
+        "   `p_ES`        tinyint(1) NOT NULL,"
+        "   `p_US`        tinyint(1) NOT NULL,"
+        "   `p_MX`        tinyint(1) NOT NULL,"
+        "   `p_CO`        tinyint(1) NOT NULL,"
+        "   `p_rank`      int(11) NOT NULL,"
+        "   PRIMARY KEY (`id`)"
+        ")  ENGINE = InnoDB DEFAULT CHARSET=latin1")
+    TABLES['tag_query3'] = (
+        "CREATE TABLE   `tag_query3` ("
+        "   `text_id`   int(11) NOT NULL AUTO_INCREMENT,"
+        "   `p_site`      varchar(20) NOT NULL,"
+        "   `p_ES`        tinyint(1) NOT NULL,"
+        "   `p_US`        tinyint(1) NOT NULL,"
+        "   `p_MX`        tinyint(1) NOT NULL,"
+        "   `p_CO`        tinyint(1) NOT NULL,"
+        "   `p_rank`      int(11) NOT NULL,"
+        "   `p_published` timestamp NOT NULL,"
+        "   `p_type`      enum('normal', 'ecommerce', 'slideshow', 'video', 'duplicate', 'branded_club', 'brand_article', 'brand_article_video', 'longform', 'reposted_slideshow') NOT NULL DEFAULT 'normal',"
+        "   PRIMARY KEY (`id`)"
+        ")  ENGINE = InnoDB DEFAULT CHARSET=latin1")
+
     for name, dbq in TABLES.items():
         try:
             print("Creating table {}: ".format(name), end = '')

main.py

@@ -3,7 +3,7 @@
 from tagsPopulator import process as tagsPopulator
 from postsPopulator import process as postsPopulator
 from tagsRelationGenerator import process as tagsRelationGenerator
-from rankGenerator import process as rankGenerator
+from tagQueryPopulator import process as tagQueryPopulator
 from config import siteConfig
 from faker import Faker
 from timer import Timer, displayTimer
@@ -24,6 +24,8 @@ def process():
         displayTimer(timer.get_time_hhmmss(), 'Elapsed')
     tagsRelationGenerator()
     displayTimer(timer.get_time_hhmmss(), 'Elapsed')
+    tagQueryPopulator()
+    displayTimer(timer.get_time_hhmmss(), 'Elapsed')
     print("=====================================================")
     print("Mock Database Created")
     print("=====================================================")

postsPopulator.py

@@ -1,8 +1,7 @@
 import pymysql.cursors
 from random import randint
 from faker import Faker
-from config import dbConfig, postTypeChoice, postsPopulatorConfig, fakeText, countryChoice, specialChoice
-# from timer import Timestamp
+from config import dbConfig, postTypeChoice, postsPopulatorConfig, fakeText, countryChoice, specialChoice, randomRank
 
 def batchInsertPosts(posts):
     query = ",".join(posts)
@@ -17,19 +16,21 @@ def batchInsertPosts(posts):
         "   `CO`,"
         "   `type`,"
         "   `url`,"
-        "   `special`"
+        "   `special`,"
+        "   `rank`"
         ")  VALUES  " + query)
     cursor.execute(query)
     connection.commit()
 
 def insertPosts():
+    rankCount = 0
     posts = []
     countriesWeighted = 0
+    rankConfig = randomRank()
     for numberOfPosts in range(1, (postsPopulatorConfig['postsCount'] + 1)):
-        fakeTextStart = randint(0,997000)
-        fakeTextEnd   = fakeTextStart + 3000
+        fakeTextStart = randint(0,998000)
+        fakeTextEnd   = fakeTextStart + 2000
         text      = fakeText[fakeTextStart:fakeTextEnd] + fake.pystr(max_chars = 20)
-        # published = timestamp.random()
         published = fake.date_time_between(start_date = "-6y", end_date = "now")
         countriesWeighted = countryChoice(countriesWeighted)
         ES        = countriesWeighted[0]
@@ -39,7 +40,9 @@ def insertPosts():
         postType  = postTypeChoice()
         url       = fake.uri() + fake.pystr(max_chars = 10)
         special   = specialChoice()
-        posts.append("('{}', '{}', '{}', {}, {}, {}, {}, '{}', '{}', {})"
+        rank      = rankConfig[rankCount]
+        rankCount += 1
+        posts.append("('{}', '{}', '{}', {}, {}, {}, {}, '{}', '{}', {}, {})"
             .format(
                 site,
                 text,
@@ -50,7 +53,8 @@ def insertPosts():
                 CO,
                 postType,
                 url,
-                special))
+                special,
+                rank))
         if (numberOfPosts % postsPopulatorConfig['batchSize']) == 0:
             batchInsertPosts(posts)
             posts = []
@@ -61,12 +65,11 @@ def insertPosts():
     print("Posts Insertion completed for {} posts".format(numberOfPosts))
 
 def process(postSite):
-    global connection, cursor, fake#, timestamp
+    global connection, cursor, fake
     global site
     connection = pymysql.connect(**dbConfig)
     cursor = connection.cursor()
     fake = Faker()
-    # timestamp = Timestamp()
     site = postSite
     print("=====================================================")
     print("Generating Post data for ---> {}".format(site))

tagQueryPopulator.py

@@ -0,0 +1,31 @@
+import pymysql.cursors
+import config
+
+def tagQueryPopulate():
+    query = "INSERT tag_query1 (p_site, p_ES, p_US, p_MX, p_CO, p_published) SELECT site, ES, US, MX, CO, published FROM wp_posts ORDER BY id ASC"
+    cursor.execute(query)
+    connection.commit()
+    print("Table 1 Populated")
+    query = "INSERT tag_query2 (p_site, p_ES, p_US, p_MX, p_CO, p_rank) SELECT site, ES, US, MX, CO, rank FROM wp_posts ORDER BY id ASC"
+    cursor.execute(query)
+    connection.commit()
+    print("Table 2 Populated")
+    query = "INSERT tag_query3 (p_site, p_ES, p_US, p_MX, p_CO, p_rank, p_published, p_type) SELECT site, ES, US, MX, CO, rank, published, type FROM wp_posts ORDER BY id ASC"
+    cursor.execute(query)
+    connection.commit() 
+    print("Table 3 Populated")
+    query = "INSERT text (p_text) SELECT text FROM wp_posts ORDER BY id ASC"
+    cursor.execute(query)
+    connection.commit() 
+    print("Table 4 Populated")
+
+def process():
+    global connection, cursor
+    connection = pymysql.connect(**config.dbConfig)
+    cursor = connection.cursor()
+    print("=====================================================")
+    print("Populating Tag Query Tables")
+    tagQueryPopulate()
+    print("=====================================================")
+    cursor.close()
+    connection.close()