Rehearsal Bugfixes

app/user.py

@@ -25,7 +25,7 @@ def post(self):
         meta = data.get('meta', None)
         user = User(userid, publickey_pem, privatekey_pem=privatekey_pem,
                     services=data['services'], meta=meta)
-        ioloop.IOLoop.current().add_callback(partial(userprocess, user))
+        ioloop.IOLoop.current().add_callback(partial(userprocess, [user]))
         self.api_response("OK")
 
 
@@ -37,6 +37,7 @@ def get(self):
         shares = self.get_arguments('share')
         passphrase = self.get_argument('passphrase', None)
         reprocess = bool(self.get_argument('reprocess', None) is not None)
+        scrape_cache = not bool(self.get_argument('scrape_cache', None) is not None)
         ticket_api = CONFIG.get('ticket_api')
 
         # we could also just pass the raw arguments, but this is more explicit
@@ -54,13 +55,15 @@ def get(self):
         show_data = json.loads(show_data_raw.body)
         show_date = show_data['data']['date']
         users_added = []
+        users_to_process = []
         for user_data in show_data['data']['users']:
             userid = user_data.pop('id')
             perms = yield exhibitperms.get_permissions(userid)
             if perms and not reprocess:
                 users_added.append({'userid': userid,
                                     'permissions': perms,
                                     'process': False})
+                continue
             publickey = user_data['publickey']
             privatekey = user_data.get('privatekey')
             meta = user_data.get('meta') or {}
@@ -69,7 +72,10 @@ def get(self):
             user = User(userid, publickey, services=user_data['services'],
                         privatekey_pem=privatekey, meta=meta)
             users_added.append({'userid': userid, 'process': True})
-            ioloop.IOLoop.current().add_callback(partial(userprocess, user))
+            users_to_process.append(user)
+        ioloop.IOLoop.current().spawn_callback(
+            partial(userprocess, users_to_process, scrape_cache)
+        )
         return self.api_response(users_added)
 
 

lib/processors/__init__.py

@@ -20,8 +20,9 @@
 logger = logging.getLogger("processor.process")
 
 processors = [
-    Pr0nProcessor(),
+    DebugProcessor(),
     TruthProcessor(),
+    Pr0nProcessor(),
     MirrorProcessor(),
     NewsProcessor(),
     OwnupProcessor(),
@@ -33,7 +34,6 @@
     RomanceProcessor(),
     TOSProcessor(),
     DeleteProcessor(),
-    DebugProcessor(),
 ]
 
 

lib/processors/ameliaprocessor.py

@@ -9,6 +9,7 @@
 
 class AmeliaProcessor(BaseProcessor):
     name = 'amelia_processor'
+    auth = False
 
     def __init__(self):
         super().__init__()

lib/processors/data/mentalwords.txt

@@ -30,6 +30,8 @@
  worse
  upsetting
  easy
+ best
+ worst
  boredom
  confusion
  confused

lib/processors/data/partywords.txt

@@ -37,4 +37,5 @@ trees
 marijuana
 cannibas
 stoned
-420
+420
+dafuq

lib/processors/data/positiveworkwords.txt

@@ -12,4 +12,5 @@ still at work
 love my boss
 best boss
 finished my homework
-allnighter
+allnighter
+working

lib/processors/debugprocessor.py

@@ -1,8 +1,5 @@
 from tornado import gen
-import pickle
-import os
 
-from ..config import CONFIG
 from .lib.baseprocessor import BaseProcessor
 
 
@@ -14,12 +11,10 @@ def process(self, user_data):
         """
         Save user data for inspection
         """
-        if CONFIG.get('_mode') != 'dev':
-            return False
-        filename = "./data/debug/{}.pkl".format(user_data.userid)
-        os.makedirs('./data/debug', exist_ok=True)
-        with open(filename, 'wb+') as fd:
-            pickle.dump(user_data.data, fd)
-        self.logger.info("Saved user {} to {}".format(user_data.userid,
-                                                      filename))
+        self.logger.info("Saving user scrape data")
+        self.save_user_blob(user_data.data, user_data)
+        self.logger.info("Saved user: " + user_data.userid)
         return True
+
+    def load_scrape(self, user_data):
+        return self.load_user_blob(user_data)

lib/processors/interviewprocessor.py

@@ -175,8 +175,9 @@ def process(self, user_data):
             pos_interests = self.scan_pos_interests(
                 pos_interests, likes)
 
-        testit = pos_quotes + neg_quotes + pos_interests + neg_interests + pos_events + neg_events
-        if len(testit) <= 2:
+        num_quotes = sum(len(quotes) for quotes in (pos_quotes, neg_quotes,
+                         pos_interests, neg_interests, pos_events, neg_events))
+        if num_quotes <= 2:
             return False
 
         interview_data = self.build_data(interview_data, pos_quotes, neg_quotes,

lib/processors/lib/baseprocessor.py

@@ -72,7 +72,7 @@ def register_handlers(self):
 
     def save_user_blob(self, blob, user):
         filedata = dict(name=self.name, uid=user.userid)
-        os.makedirs("./data/{name}/user".format(**filedata), exist_ok=True)
+        os.makedirs("./data/{name}/user/".format(**filedata), exist_ok=True)
         if CONFIG.get('_mode') == 'dev':
             filename = "./data/{name}/user/{uid}.pkl".format(**filedata)
             with open(filename, 'wb+') as fd:

lib/processors/lib/keyword_filters.py

@@ -0,0 +1,57 @@
+import re
+
+
+def is_great_quote(text, keywords):
+    text = text.lower()
+    words = set(text.split())
+    if len(words) < 3:
+        return False
+    if any(len(w) > 13 or 'http' in w
+            for w in words):
+        return False
+    if "@" in text:
+        return False
+    for keyword in keywords:
+        if keyword in words:
+            return True
+    return False
+
+
+def process_post(text, keywords):
+    # If there is enough perm quotes, quit.
+    sentences = re.split(r' *[\.\?!][\'"\)\]]* *', text)
+    for sentence in map(str.strip, sentences):
+        if is_great_quote(sentence, keywords):
+            yield sentence
+
+
+def process_facebook(user_data, keywords):
+    if user_data.data.get('fbtext', None):
+        fbtext = user_data.data['fbtext']
+        fbposts = fbtext.get('text') or []
+        for post in fbposts:
+            yield from process_post(post['text'], keywords)
+
+
+def process_twitter(user_data, keywords):
+    if user_data.data.get('twitter', None):
+        twitter = user_data.data['twitter']
+        tweets = twitter.get('tweets') or []
+        for post in tweets:
+            yield from process_post(post, keywords)
+
+
+def process_reddit(user_data, keywords):
+    if user_data.data.get('reddit', None):
+        reddit = user_data.data['reddit']
+        posts = reddit.get('text') or []
+        for post in posts:
+            yield from process_post(post['body'], keywords)
+
+
+def process_gmail(user_data, keywords):
+    if user_data.data.get('gmail'):
+        gmail = user_data.data['gmail']
+        snippets = gmail.get('snippets') or []
+        for post in snippets:
+            yield from process_post(post, keywords)

lib/processors/mentalhealthprocessor.py

@@ -1,64 +1,38 @@
 from tornado import gen
-import itertools as IT
+
+import random
 
 from .lib.handler import process_api_handler
 from .lib.baseprocessor import BaseProcessor
+from .lib import keyword_filters
 
 
 class MentalHealthProcessor(BaseProcessor):
     name = 'mental_health'
-    limit = 200
+    num_quotes = 200
 
     def __init__(self):
         super().__init__()
         self.keywords = self.load_keywords('mentalwords.txt')
 
-    def is_good_quote(self, text):
-        text = text.lower()
-        for word in self.keywords:
-            if word in text:
-                return True
-        return False
-
-    def process_facebook(self, user_data):
-        if user_data.data.get('fbtext'):
-            fbtext = user_data.data['fbtext']
-            fbposts = fbtext.get('text', [])
-            for post in fbposts:
-                yield post['text']
-
-    def process_twitter(self, user_data):
-        if user_data.data.get('twitter'):
-            twitter = user_data.data['twitter']
-            tweets = twitter.get('tweets', [])
-            for post in tweets:
-                yield post
-
-    def process_reddit(self, user_data):
-        if user_data.data.get('reddit'):
-            reddit = user_data.data['reddit']
-            posts = reddit.get('text', [])
-            for post in posts:
-                yield post['body']
-
-    def process_gmail(self, user_data):
-        if user_data.data.get('gmail'):
-            gmail = user_data.data['gmail']
-            for post in gmail.get('snippet', []):
-                yield post
-
     @gen.coroutine
     def process(self, user_data):
         self.logger.info("Processing user: {}".format(user_data.userid))
-        quote_candidates = IT.chain(self.process_facebook(user_data),
-                                    self.process_twitter(user_data),
-                                    self.process_reddit(user_data),
-                                    self.process_gmail(user_data))
-        quotes = list(IT.islice(quote_candidates, self.limit))
+        quotes = []
+        quotes += list(keyword_filters.process_facebook(user_data,
+                                                        self.keywords))
+        quotes += list(keyword_filters.process_twitter(user_data,
+                                                       self.keywords))
+        quotes += list(keyword_filters.process_reddit(user_data,
+                                                      self.keywords))
+        quotes += list(keyword_filters.process_gmail(user_data,
+                                                     self.keywords))
         self.logger.debug("User {}: {} quotes".format(user_data.userid,
                                                       len(quotes)))
-        if len(quotes) < 0:
+        if len(quotes) < 5:
             return False
+        if len(quotes) > self.num_quotes:
+            quotes = random.sample(quotes, self.num_quotes)
         self.save_user_blob(quotes, user_data)
         return True
 

lib/processors/ownupprocessor.py

@@ -2,90 +2,45 @@
 
 from .lib.handler import process_api_handler
 from .lib.baseprocessor import BaseProcessor
+from .lib import keyword_filters
 
 import random
 
 
 class OwnupProcessor(BaseProcessor):
     name = 'ownup'
-    limit = 10
+    num_quotes = 100
 
     def __init__(self):
         super().__init__()
         self.keywords = self.load_keywords('ownup.txt')
 
-    def is_great_quote(self, text):
-        text = text.lower()
-        for word in self.keywords:
-            if text.find(word) >= 0:
-                return True
-        return False
-
-    """ Return True if processing should continue
-    """
-    def process_post(self, text, temp, perm):
-        # If there is enough perm quotes, quit.
-        if len(perm) >= self.limit:
-            return False
-        sentences = text.split('.')
-        for sentence in sentences:
-            if self.is_great_quote(sentence):
-                perm.append(sentence)
-                random.shuffle(perm)
-            elif len(sentence) > 15:
-                temp.append(sentence)
-                random.shuffle(perm)
-        return True
-
-    def process_facebook(self, user_data, temp, perm):
-        if user_data.data.get('fbtext', None):
-            fbtext = user_data.data.get('fbtext')
-            if not fbtext or len(perm) >= self.limit:
-                return
-            fbposts = fbtext['text']
-            for post in fbposts:
-                if not self.process_post(post['text'], temp, perm):
-                    return
-
-    def process_twitter(self, user_data, temp, perm):
-        if user_data.data.get('twitter', None):
-            twitter = user_data.data.get('twitter', None)
-            if not twitter or len(perm) >= self.limit:
-                return
-            tweets = twitter['tweets']
-            for post in tweets:
-                if not self.process_post(post, temp, perm):
-                    return
-
-    def process_reddit(self, user_data, temp, perm):
-        if user_data.data.get('reddit', None):
-            reddit = user_data.data.get('reddit', None)
-            if not reddit or len(perm) >= self.limit:
-                return
-            posts = reddit['text']
-            for post in posts:
-                if not self.process_post(post['body'], temp, perm):
-                    return
-
     @gen.coroutine
     def process(self, user_data):
-        self.logger.info("[OU] Processing user: {}".format(user_data.userid))
-        temp = []
-        perm = []
-        self.process_facebook(user_data, temp, perm)
-        self.process_twitter(user_data, temp, perm)
-        self.process_reddit(user_data, temp, perm)
-        while len(perm) < self.limit and len(temp) > 1:
-            perm.append(temp.pop())
-        if len(perm) < 3:
+        self.logger.info("Processing user: {}".format(user_data.userid))
+        quotes = []
+        quotes += list(keyword_filters.process_facebook(user_data,
+                                                        self.keywords))
+        quotes += list(keyword_filters.process_twitter(user_data,
+                                                       self.keywords))
+        quotes += list(keyword_filters.process_reddit(user_data,
+                                                      self.keywords))
+        self.logger.debug("User %s has %d quotes", user_data.userid, len(quotes))
+        if not quotes:
             return False
-        blob = {'name': user_data.meta['name'], 'quotes': perm}
+        if len(quotes) > self.num_quotes:
+            quotes = random.sample(quotes, self.num_quotes)
+        blob = {'name': user_data.meta['name'],
+                'quotes': quotes}
         self.save_user_blob(blob, user_data)
         return True
 
     @gen.coroutine
     def get_quotes(self, user, request):
-        return self.load_user_blob(user)
+        data = self.load_user_blob(user)
+        if len(data['quotes']) > 10:
+            data['quotes'] = random.sample(data['quotes'], 10)
+        return data
 
     @process_api_handler
     def register_handlers(self):