From e60b27e0383e9ef0dc732f4858584f1aca847c9b Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 00:33:32 -0600 Subject: [PATCH 01/30] Cleaned up ownup/mentalhealth/interview --- lib/processors/ameliaprocessor.py | 1 + lib/processors/interviewprocessor.py | 5 +- lib/processors/lib/keyword_filters.py | 57 +++++++++++++++++ lib/processors/mentalhealthprocessor.py | 58 +++++------------ lib/processors/ownupprocessor.py | 83 ++++++------------------- 5 files changed, 95 insertions(+), 109 deletions(-) create mode 100644 lib/processors/lib/keyword_filters.py diff --git a/lib/processors/ameliaprocessor.py b/lib/processors/ameliaprocessor.py index c2edcdc..cec3307 100644 --- a/lib/processors/ameliaprocessor.py +++ b/lib/processors/ameliaprocessor.py @@ -9,6 +9,7 @@ class AmeliaProcessor(BaseProcessor): name = 'amelia_processor' + auth = False def __init__(self): super().__init__() diff --git a/lib/processors/interviewprocessor.py b/lib/processors/interviewprocessor.py index 31a549e..8514086 100644 --- a/lib/processors/interviewprocessor.py +++ b/lib/processors/interviewprocessor.py @@ -175,8 +175,9 @@ def process(self, user_data): pos_interests = self.scan_pos_interests( pos_interests, likes) - testit = pos_quotes + neg_quotes + pos_interests + neg_interests + pos_events + neg_events - if len(testit) <= 2: + num_quotes = sum(len(quotes) for quotes in (pos_quotes, neg_quotes, + pos_interests, neg_interests, pos_events, neg_events)) + if num_quotes <= 2: return False interview_data = self.build_data(interview_data, pos_quotes, neg_quotes, diff --git a/lib/processors/lib/keyword_filters.py b/lib/processors/lib/keyword_filters.py new file mode 100644 index 0000000..05481e0 --- /dev/null +++ b/lib/processors/lib/keyword_filters.py @@ -0,0 +1,57 @@ +import re + + +def is_great_quote(text, keywords): + text = text.lower() + words = set(text.split()) + if len(words) < 3: + return False + if any(len(w) > 10 or 'http' in w + for w in words): + return False + if "@" in text: + return False + for keyword in keywords: + if keyword in words: + return True + return False + + +def process_post(text, keywords): + # If there is enough perm quotes, quit. + sentences = re.split(r' *[\.\?!][\'"\)\]]* *', text) + for sentence in map(str.strip, sentences): + if is_great_quote(sentence, keywords): + yield sentence + + +def process_facebook(user_data, keywords): + if user_data.data.get('fbtext', None): + fbtext = user_data.data['fbtext'] + fbposts = fbtext.get('text') or [] + for post in fbposts: + yield from process_post(post['text'], keywords) + + +def process_twitter(user_data, keywords): + if user_data.data.get('twitter', None): + twitter = user_data.data['twitter'] + tweets = twitter.get('tweets') or [] + for post in tweets: + yield from process_post(post, keywords) + + +def process_reddit(user_data, keywords): + if user_data.data.get('reddit', None): + reddit = user_data.data['reddit'] + posts = reddit.get('text') or [] + for post in posts: + yield from process_post(post['body'], keywords) + + +def process_gmail(user_data, keywords): + if user_data.data.get('gmail'): + gmail = user_data.data['gmail'] + snippets = gmail.get('snippets') or [] + for post in snippets: + yield from process_post(post, keywords) diff --git a/lib/processors/mentalhealthprocessor.py b/lib/processors/mentalhealthprocessor.py index 7b55c50..9d0586d 100644 --- a/lib/processors/mentalhealthprocessor.py +++ b/lib/processors/mentalhealthprocessor.py @@ -1,64 +1,38 @@ from tornado import gen -import itertools as IT + +import random from .lib.handler import process_api_handler from .lib.baseprocessor import BaseProcessor +from .lib import keyword_filters class MentalHealthProcessor(BaseProcessor): name = 'mental_health' - limit = 200 + num_quotes = 200 def __init__(self): super().__init__() self.keywords = self.load_keywords('mentalwords.txt') - def is_good_quote(self, text): - text = text.lower() - for word in self.keywords: - if word in text: - return True - return False - - def process_facebook(self, user_data): - if user_data.data.get('fbtext'): - fbtext = user_data.data['fbtext'] - fbposts = fbtext.get('text', []) - for post in fbposts: - yield post['text'] - - def process_twitter(self, user_data): - if user_data.data.get('twitter'): - twitter = user_data.data['twitter'] - tweets = twitter.get('tweets', []) - for post in tweets: - yield post - - def process_reddit(self, user_data): - if user_data.data.get('reddit'): - reddit = user_data.data['reddit'] - posts = reddit.get('text', []) - for post in posts: - yield post['body'] - - def process_gmail(self, user_data): - if user_data.data.get('gmail'): - gmail = user_data.data['gmail'] - for post in gmail.get('snippet', []): - yield post - @gen.coroutine def process(self, user_data): self.logger.info("Processing user: {}".format(user_data.userid)) - quote_candidates = IT.chain(self.process_facebook(user_data), - self.process_twitter(user_data), - self.process_reddit(user_data), - self.process_gmail(user_data)) - quotes = list(IT.islice(quote_candidates, self.limit)) + quotes = [] + quotes += list(keyword_filters.process_facebook(user_data, + self.keywords)) + quotes += list(keyword_filters.process_twitter(user_data, + self.keywords)) + quotes += list(keyword_filters.process_reddit(user_data, + self.keywords)) + quotes += list(keyword_filters.process_gmail(user_data, + self.keywords)) self.logger.debug("User {}: {} quotes".format(user_data.userid, len(quotes))) - if len(quotes) < 0: + if len(quotes) < 5: return False + if len(quotes) > self.num_quotes: + quotes = random.sample(quotes, self.num_quotes) self.save_user_blob(quotes, user_data) return True diff --git a/lib/processors/ownupprocessor.py b/lib/processors/ownupprocessor.py index 748bd9a..4935e84 100644 --- a/lib/processors/ownupprocessor.py +++ b/lib/processors/ownupprocessor.py @@ -2,90 +2,43 @@ from .lib.handler import process_api_handler from .lib.baseprocessor import BaseProcessor +from .lib import keyword_filters import random class OwnupProcessor(BaseProcessor): name = 'ownup' - limit = 10 + num_quotes = 100 def __init__(self): super().__init__() self.keywords = self.load_keywords('ownup.txt') - def is_great_quote(self, text): - text = text.lower() - for word in self.keywords: - if text.find(word) >= 0: - return True - return False - - """ Return True if processing should continue - """ - def process_post(self, text, temp, perm): - # If there is enough perm quotes, quit. - if len(perm) >= self.limit: - return False - sentences = text.split('.') - for sentence in sentences: - if self.is_great_quote(sentence): - perm.append(sentence) - random.shuffle(perm) - elif len(sentence) > 15: - temp.append(sentence) - random.shuffle(perm) - return True - - def process_facebook(self, user_data, temp, perm): - if user_data.data.get('fbtext', None): - fbtext = user_data.data.get('fbtext') - if not fbtext or len(perm) >= self.limit: - return - fbposts = fbtext['text'] - for post in fbposts: - if not self.process_post(post['text'], temp, perm): - return - - def process_twitter(self, user_data, temp, perm): - if user_data.data.get('twitter', None): - twitter = user_data.data.get('twitter', None) - if not twitter or len(perm) >= self.limit: - return - tweets = twitter['tweets'] - for post in tweets: - if not self.process_post(post, temp, perm): - return - - def process_reddit(self, user_data, temp, perm): - if user_data.data.get('reddit', None): - reddit = user_data.data.get('reddit', None) - if not reddit or len(perm) >= self.limit: - return - posts = reddit['text'] - for post in posts: - if not self.process_post(post['body'], temp, perm): - return - @gen.coroutine def process(self, user_data): - self.logger.info("[OU] Processing user: {}".format(user_data.userid)) - temp = [] - perm = [] - self.process_facebook(user_data, temp, perm) - self.process_twitter(user_data, temp, perm) - self.process_reddit(user_data, temp, perm) - while len(perm) < self.limit and len(temp) > 1: - perm.append(temp.pop()) - if len(perm) < 3: + self.logger.info("Processing user: {}".format(user_data.userid)) + quotes = [] + quotes += list(keyword_filters.process_facebook(user_data, + self.keywords)) + quotes += list(keyword_filters.process_twitter(user_data, + self.keywords)) + quotes += list(keyword_filters.process_reddit(user_data, + self.keywords)) + if len(quotes) < 10: return False - blob = {'name': user_data.meta['name'], 'quotes': perm} + if len(quotes) > self.num_quotes: + quotes = random.sample(quotes, self.num_quotes) + blob = {'name': user_data.meta['name'], + 'quotes': quotes} self.save_user_blob(blob, user_data) return True @gen.coroutine def get_quotes(self, user, request): - return self.load_user_blob(user) + data = self.load_user_blob(user) + data['quotes'] = random.sample(data['quotes'], 10) + return data @process_api_handler def register_handlers(self): From 1962262679ff6cda289ed4b6d206f7a721a074a7 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 00:50:59 -0600 Subject: [PATCH 02/30] longer words --- lib/processors/lib/keyword_filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/processors/lib/keyword_filters.py b/lib/processors/lib/keyword_filters.py index 05481e0..8c74a2e 100644 --- a/lib/processors/lib/keyword_filters.py +++ b/lib/processors/lib/keyword_filters.py @@ -6,7 +6,7 @@ def is_great_quote(text, keywords): words = set(text.split()) if len(words) < 3: return False - if any(len(w) > 10 or 'http' in w + if any(len(w) > 13 or 'http' in w for w in words): return False if "@" in text: From 1ac8fa7442a682fa2267f0e0d4c658c9690fa07c Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 01:16:59 -0600 Subject: [PATCH 03/30] trying to fix truth --- lib/processors/truthprocessor.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index 30e7720..b5f2495 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -11,6 +11,7 @@ class TruthProcessor(BaseProcessor): name = 'truth_processor' data = {} + num_items = 100 def __init__(self): super().__init__() @@ -83,16 +84,6 @@ def _common_and_lie(self, items): lie, _ = random.choice(top_10[1:]) return best_item, lie - def fill_truths(self, truestuff): - while len(truestuff) < self.truths: - truestuff.append(random.choice(self.realfacts)) - return truestuff - - def fill_lies(self, lies): - while len(lies) < (15 - self.truths): - lies.append(random.choice(self.fakefacts)) - return lies - def percentage_check(self, word, text_list, thresh, fact_str, facts, lies): freq = self.get_percentage(word, text_list) if freq > thresh: @@ -225,8 +216,10 @@ def process(self, user_data): except IndexError: pass - truth_data['true'] = self.fill_truths(truth_data['true']) - truth_data['false'] = self.fill_lies(truth_data['false']) + random.shuffle(truth_data['true']) + random.shuffle(truth_data['false']) + truth_data['true'] = truth_data['true'][:self.num_items] + truth_data['false'] = truth_data['false'][:self.num_items] self.save_user_blob(truth_data, user_data) self.logger.info("Saved truth game data") @@ -241,6 +234,10 @@ def truth_grab(self, user, request): Returns relevant data that the exhibits may want to know """ data = self.load_user_blob(user) + if len(data['true']) > 10: + data['true'] = random.sample(data['true'], 10) + if len(data['false']) > 10: + data['false'] = random.sample(data['false'], 10) return data @process_api_handler From 0a02a65e7ba2f37ca7635f6cf6c60e265aebc2fb Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 09:33:21 -0600 Subject: [PATCH 04/30] save scrape data --- app/user.py | 3 ++- lib/processors/debugprocessor.py | 16 +++++----------- lib/processors/lib/baseprocessor.py | 16 +++++++++------- lib/userprocess.py | 11 +++-------- 4 files changed, 19 insertions(+), 27 deletions(-) diff --git a/app/user.py b/app/user.py index c3b9085..14fb53a 100644 --- a/app/user.py +++ b/app/user.py @@ -37,6 +37,7 @@ def get(self): shares = self.get_arguments('share') passphrase = self.get_argument('passphrase', None) reprocess = bool(self.get_argument('reprocess', None) is not None) + scrape_cache = not bool(self.get_argument('scrape_cache', None) is not None) ticket_api = CONFIG.get('ticket_api') # we could also just pass the raw arguments, but this is more explicit @@ -69,7 +70,7 @@ def get(self): user = User(userid, publickey, services=user_data['services'], privatekey_pem=privatekey, meta=meta) users_added.append({'userid': userid, 'process': True}) - ioloop.IOLoop.current().add_callback(partial(userprocess, user)) + ioloop.IOLoop.current().add_callback(partial(userprocess, user, scrape_cache)) return self.api_response(users_added) diff --git a/lib/processors/debugprocessor.py b/lib/processors/debugprocessor.py index 71e1f78..6ea7933 100644 --- a/lib/processors/debugprocessor.py +++ b/lib/processors/debugprocessor.py @@ -1,8 +1,5 @@ from tornado import gen -import pickle -import os -from ..config import CONFIG from .lib.baseprocessor import BaseProcessor @@ -14,12 +11,9 @@ def process(self, user_data): """ Save user data for inspection """ - if CONFIG.get('_mode') != 'dev': - return False - filename = "./data/debug/{}.pkl".format(user_data.userid) - os.makedirs('./data/debug', exist_ok=True) - with open(filename, 'wb+') as fd: - pickle.dump(user_data.data, fd) - self.logger.info("Saved user {} to {}".format(user_data.userid, - filename)) + self.save_user_blob(user_data.data, user_data) + self.logger.info("Saved user: " + user_data.userid) return True + + def load_scrape(self, user_data): + return self.load_user_blob(user_data) diff --git a/lib/processors/lib/baseprocessor.py b/lib/processors/lib/baseprocessor.py index ea39025..1bdaa62 100644 --- a/lib/processors/lib/baseprocessor.py +++ b/lib/processors/lib/baseprocessor.py @@ -71,26 +71,28 @@ def register_handlers(self): return [] def save_user_blob(self, blob, user): - filedata = dict(name=self.name, uid=user.userid) - os.makedirs("./data/{name}/user".format(**filedata), exist_ok=True) + filedata = dict(name=self.name, uid=user.userid, + showid=user.meta.get('showid', 'misc')) + os.makedirs("./data/{name}/user/{showid}".format(**filedata), exist_ok=True) if CONFIG.get('_mode') == 'dev': - filename = "./data/{name}/user/{uid}.pkl".format(**filedata) + filename = "./data/{name}/user/{showid}/{uid}.pkl".format(**filedata) with open(filename, 'wb+') as fd: pickle.dump(blob, fd) else: blob_enc = user.encrypt_blob(blob) - filename = "./data/{name}/user/{uid}.enc".format(**filedata) + filename = "./data/{name}/user/{showid}/{uid}.enc".format(**filedata) with open(filename, 'wb+') as fd: fd.write(blob_enc) def load_user_blob(self, user): - filedata = dict(name=self.name, uid=user.userid) + filedata = dict(name=self.name, uid=user.userid, + showid=user.meta.get('showid', 'misc')) try: - filename = "./data/{name}/user/{uid}.pkl".format(**filedata) + filename = "./data/{name}/user/{showid}/{uid}.pkl".format(**filedata) with open(filename, 'rb') as fd: return pickle.load(fd) except IOError: - filename = "./data/{name}/user/{uid}.enc".format(**filedata) + filename = "./data/{name}/user/{showid}/{uid}.enc".format(**filedata) with open(filename, 'rb') as fd: blob = fd.read() return user.decrypt_blob(blob) diff --git a/lib/userprocess.py b/lib/userprocess.py index 7429541..7004dd9 100644 --- a/lib/userprocess.py +++ b/lib/userprocess.py @@ -1,19 +1,14 @@ from tornado import gen -import pickle from . import scrapers from . import processors -from .config import CONFIG @gen.coroutine -def userprocess(user): - if CONFIG.get('_mode') == 'dev': +def userprocess(user, use_cache=True): + if use_cache: try: - fname = './data/debug/{}.pkl'.format(user.userid) - with open(fname, 'rb') as fd: - user.data = pickle.load(fd) - print("Found debug data for user: ", user.userid) + user.data = processors.DebugProcessor().load_scrape(user) except IOError: pass if not user.data: From 92dc620b6b2cb2a8d692f35ba1c9253a9eb6c1a5 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 09:42:19 -0600 Subject: [PATCH 05/30] reprocessing wasn't respecting permissions --- app/user.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/user.py b/app/user.py index 14fb53a..bc9767f 100644 --- a/app/user.py +++ b/app/user.py @@ -62,6 +62,7 @@ def get(self): users_added.append({'userid': userid, 'permissions': perms, 'process': False}) + continue publickey = user_data['publickey'] privatekey = user_data.get('privatekey') meta = user_data.get('meta') or {} From a12c2a81a003e19f2251a9972916efedfa7840c8 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 11:13:51 -0600 Subject: [PATCH 06/30] reduced number of fb photos --- lib/scrapers/fbphotos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/scrapers/fbphotos.py b/lib/scrapers/fbphotos.py index 868158f..8c00890 100644 --- a/lib/scrapers/fbphotos.py +++ b/lib/scrapers/fbphotos.py @@ -15,8 +15,8 @@ class FBPhotosScraper(BaseScraper): @property def num_images_per_user(self): if CONFIG['_mode'] == 'prod': - return 1000 - return 250 + return 300 + return 50 @gen.coroutine def scrape(self, user_data): From b4e7607a0c4c3aebeb0d920559f906b7451821ae Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 12:51:21 -0600 Subject: [PATCH 07/30] got rid of showtime organization --- lib/processors/lib/baseprocessor.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/processors/lib/baseprocessor.py b/lib/processors/lib/baseprocessor.py index 1bdaa62..c2148b9 100644 --- a/lib/processors/lib/baseprocessor.py +++ b/lib/processors/lib/baseprocessor.py @@ -71,28 +71,26 @@ def register_handlers(self): return [] def save_user_blob(self, blob, user): - filedata = dict(name=self.name, uid=user.userid, - showid=user.meta.get('showid', 'misc')) - os.makedirs("./data/{name}/user/{showid}".format(**filedata), exist_ok=True) + filedata = dict(name=self.name, uid=user.userid) + os.makedirs("./data/{name}/user/".format(**filedata), exist_ok=True) if CONFIG.get('_mode') == 'dev': - filename = "./data/{name}/user/{showid}/{uid}.pkl".format(**filedata) + filename = "./data/{name}/user/{uid}.pkl".format(**filedata) with open(filename, 'wb+') as fd: pickle.dump(blob, fd) else: blob_enc = user.encrypt_blob(blob) - filename = "./data/{name}/user/{showid}/{uid}.enc".format(**filedata) + filename = "./data/{name}/user/{uid}.enc".format(**filedata) with open(filename, 'wb+') as fd: fd.write(blob_enc) def load_user_blob(self, user): - filedata = dict(name=self.name, uid=user.userid, - showid=user.meta.get('showid', 'misc')) + filedata = dict(name=self.name, uid=user.userid) try: - filename = "./data/{name}/user/{showid}/{uid}.pkl".format(**filedata) + filename = "./data/{name}/user/{uid}.pkl".format(**filedata) with open(filename, 'rb') as fd: return pickle.load(fd) except IOError: - filename = "./data/{name}/user/{showid}/{uid}.enc".format(**filedata) + filename = "./data/{name}/user/{uid}.enc".format(**filedata) with open(filename, 'rb') as fd: blob = fd.read() return user.decrypt_blob(blob) From 068bc73c52d9aed412576c94aebee31f553077f1 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 13:09:37 -0600 Subject: [PATCH 08/30] speedup scrapers --- lib/scrapers/__init__.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/scrapers/__init__.py b/lib/scrapers/__init__.py index 754ec0e..20a7371 100644 --- a/lib/scrapers/__init__.py +++ b/lib/scrapers/__init__.py @@ -30,18 +30,22 @@ ] +@gen.coroutine +def _scrape(s, user_data): + try: + s.logger.info("Starting to scrape: " + user_data.userid) + result = yield s.scrape(user_data) + if not result: + s.logger.info("Scraped no data: " + user_data.userid) + else: + s.logger.info("Scraped data sucessfully: " + user_data.userid) + except Exception: + result = None + s.logger.exception("Scraper failed on user: " + user_data.userid) + return result + + @gen.coroutine def scrape(user_data): - data = {} - for s in scrapers: - try: - s.logger.info("Starting to scrape: " + user_data.userid) - data[s.name] = yield s.scrape(user_data) - if not data[s.name]: - s.logger.info("Scraped no data: " + user_data.userid) - else: - s.logger.info("Scraped data sucessfully: " + user_data.userid) - except Exception: - data[s.name] = None - s.logger.exception("Scraper failed on user: " + user_data.userid) + data = yield {s.name: scrape(s, user_data) for s in scrapers} return data From 293861eed32b031e82dd2b725645cf6474812105 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 13:44:27 -0600 Subject: [PATCH 09/30] more yielding to ioloop --- app/user.py | 2 +- lib/scrapers/__init__.py | 2 +- lib/scrapers/gmailscrape.py | 5 ++++- lib/scrapers/twitter.py | 2 ++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/app/user.py b/app/user.py index bc9767f..975732e 100644 --- a/app/user.py +++ b/app/user.py @@ -71,7 +71,7 @@ def get(self): user = User(userid, publickey, services=user_data['services'], privatekey_pem=privatekey, meta=meta) users_added.append({'userid': userid, 'process': True}) - ioloop.IOLoop.current().add_callback(partial(userprocess, user, scrape_cache)) + ioloop.IOLoop.current().spawn_callback(partial(userprocess, user, scrape_cache)) return self.api_response(users_added) diff --git a/lib/scrapers/__init__.py b/lib/scrapers/__init__.py index 20a7371..94e6696 100644 --- a/lib/scrapers/__init__.py +++ b/lib/scrapers/__init__.py @@ -47,5 +47,5 @@ def _scrape(s, user_data): @gen.coroutine def scrape(user_data): - data = yield {s.name: scrape(s, user_data) for s in scrapers} + data = yield {s.name: _scrape(s, user_data) for s in scrapers} return data diff --git a/lib/scrapers/gmailscrape.py b/lib/scrapers/gmailscrape.py index 7a9d7ba..5616d07 100644 --- a/lib/scrapers/gmailscrape.py +++ b/lib/scrapers/gmailscrape.py @@ -65,6 +65,7 @@ def get_content(self, raw): email_dict['body'] = body return email_dict + @gen.coroutine def paginate_messages(self, service, response, max_results=None): threads = [] if 'messages' in response: @@ -80,6 +81,7 @@ def paginate_messages(self, service, response, max_results=None): threads.append(i['threadId']) if max_results and len(threads) >= max_results: break + yield gen.sleep(0) return threads def get_recipient(self, email_data): @@ -167,7 +169,7 @@ def scrape(self, user_data): res = gmail.users().messages().list( userId='me', q='in:sent {0}'.format(token)).execute() - thread_ids_per_token[token] = self.paginate_messages( + thread_ids_per_token[token] = yield self.paginate_messages( gmail, res, max_results=self.num_threads @@ -188,4 +190,5 @@ def scrape(self, user_data): data['text'].append(body) data['snippets'].append(snippet) data['people'].append(people) + yield gen.sleep(0) return data diff --git a/lib/scrapers/twitter.py b/lib/scrapers/twitter.py index 459276f..728dbaf 100644 --- a/lib/scrapers/twitter.py +++ b/lib/scrapers/twitter.py @@ -43,6 +43,7 @@ def all_following(self, api): 'description': friend_meta.description, } data.append(user) + yield gen.sleep(0) return data @gen.coroutine @@ -57,6 +58,7 @@ def all_tweets(self, api, max_tweets): break data.extend(t.text for t in tweets) max_id = tweets.max_id + yield gen.sleep(0) return data @gen.coroutine From cfdb77066c8cc5bbdd8efac58d5442d47d4a2a39 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 13:48:19 -0600 Subject: [PATCH 10/30] less fb profile pics and better fb pagination output --- lib/scrapers/fbphotos.py | 2 +- lib/scrapers/lib/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/scrapers/fbphotos.py b/lib/scrapers/fbphotos.py index 8c00890..caae80e 100644 --- a/lib/scrapers/fbphotos.py +++ b/lib/scrapers/fbphotos.py @@ -75,7 +75,7 @@ def get_friends_profile(self, graph): 'feed', fields='from' ), - max_results=None, + max_results=self.num_images_per_user, ) friends = {d['from']['id']: d['from']['name'] for d in friends_raw} diff --git a/lib/scrapers/lib/utils.py b/lib/scrapers/lib/utils.py index 897fa13..5a8008d 100644 --- a/lib/scrapers/lib/utils.py +++ b/lib/scrapers/lib/utils.py @@ -43,6 +43,7 @@ def apiclient_paginate(resource, action, params, http=None, max_results=500): def facebook_paginate(data, max_results=500): paginated_data = [] while True: + logger.debug("Paginating facebook data: %d/%d", len(paginated_data), max_results) paginated_data.extend(data['data']) if max_results is not None and len(paginated_data) >= max_results: break From b255abb9b6525f799facabc0ca604eeb0e216e47 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 13:57:31 -0600 Subject: [PATCH 11/30] prioritize the debug --- lib/processors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/processors/__init__.py b/lib/processors/__init__.py index c7b07cb..f956735 100644 --- a/lib/processors/__init__.py +++ b/lib/processors/__init__.py @@ -20,6 +20,7 @@ logger = logging.getLogger("processor.process") processors = [ + DebugProcessor(), Pr0nProcessor(), TruthProcessor(), MirrorProcessor(), @@ -33,7 +34,6 @@ RomanceProcessor(), TOSProcessor(), DeleteProcessor(), - DebugProcessor(), ] From a491f691a81a2c367e2cc0138e6cb6440c8f744f Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 15:31:17 -0600 Subject: [PATCH 12/30] lowered the bar for ownup --- lib/processors/debugprocessor.py | 1 + lib/processors/ownupprocessor.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/processors/debugprocessor.py b/lib/processors/debugprocessor.py index 6ea7933..71ed514 100644 --- a/lib/processors/debugprocessor.py +++ b/lib/processors/debugprocessor.py @@ -11,6 +11,7 @@ def process(self, user_data): """ Save user data for inspection """ + self.logger.info("Saving user scrape data") self.save_user_blob(user_data.data, user_data) self.logger.info("Saved user: " + user_data.userid) return True diff --git a/lib/processors/ownupprocessor.py b/lib/processors/ownupprocessor.py index 4935e84..e81e69e 100644 --- a/lib/processors/ownupprocessor.py +++ b/lib/processors/ownupprocessor.py @@ -25,7 +25,8 @@ def process(self, user_data): self.keywords)) quotes += list(keyword_filters.process_reddit(user_data, self.keywords)) - if len(quotes) < 10: + self.logger.debug("User %s has %d quotes", user_data.userid, len(quotes)) + if not quotes: return False if len(quotes) > self.num_quotes: quotes = random.sample(quotes, self.num_quotes) @@ -37,7 +38,8 @@ def process(self, user_data): @gen.coroutine def get_quotes(self, user, request): data = self.load_user_blob(user) - data['quotes'] = random.sample(data['quotes'], 10) + if len(data['quotes']) > 10: + data['quotes'] = random.sample(data['quotes'], 10) return data @process_api_handler From 4616634c928262297577b345df1368874b24942c Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 15:33:04 -0600 Subject: [PATCH 13/30] process users sequentially --- app/user.py | 8 ++++++-- lib/userprocess.py | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/app/user.py b/app/user.py index 975732e..8e7aff3 100644 --- a/app/user.py +++ b/app/user.py @@ -25,7 +25,7 @@ def post(self): meta = data.get('meta', None) user = User(userid, publickey_pem, privatekey_pem=privatekey_pem, services=data['services'], meta=meta) - ioloop.IOLoop.current().add_callback(partial(userprocess, user)) + ioloop.IOLoop.current().add_callback(partial(userprocess, [user])) self.api_response("OK") @@ -55,6 +55,7 @@ def get(self): show_data = json.loads(show_data_raw.body) show_date = show_data['data']['date'] users_added = [] + users_to_process = [] for user_data in show_data['data']['users']: userid = user_data.pop('id') perms = yield exhibitperms.get_permissions(userid) @@ -71,7 +72,10 @@ def get(self): user = User(userid, publickey, services=user_data['services'], privatekey_pem=privatekey, meta=meta) users_added.append({'userid': userid, 'process': True}) - ioloop.IOLoop.current().spawn_callback(partial(userprocess, user, scrape_cache)) + users_to_process.append(user) + ioloop.IOLoop.current().spawn_callback( + partial(userprocess, users_to_process, scrape_cache) + ) return self.api_response(users_added) diff --git a/lib/userprocess.py b/lib/userprocess.py index 7004dd9..d826d62 100644 --- a/lib/userprocess.py +++ b/lib/userprocess.py @@ -5,12 +5,13 @@ @gen.coroutine -def userprocess(user, use_cache=True): - if use_cache: - try: - user.data = processors.DebugProcessor().load_scrape(user) - except IOError: - pass - if not user.data: - user.data = yield scrapers.scrape(user) - user.process_results = yield processors.process(user) +def userprocess(users, use_cache=True): + for user in users: + if use_cache: + try: + user.data = processors.DebugProcessor().load_scrape(user) + except IOError: + pass + if not user.data: + user.data = yield scrapers.scrape(user) + user.process_results = yield processors.process(user) From 1385f6f29af73caf0aba6caecc66c897a000d529 Mon Sep 17 00:00:00 2001 From: wannabeCitizen Date: Wed, 27 Apr 2016 20:48:43 -0600 Subject: [PATCH 14/30] fixed the issue with having too little data on Truth --- lib/processors/truthprocessor.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index b5f2495..cc1259e 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -220,6 +220,14 @@ def process(self, user_data): random.shuffle(truth_data['false']) truth_data['true'] = truth_data['true'][:self.num_items] truth_data['false'] = truth_data['false'][:self.num_items] + if truth_data['true'] < 8: + missing = 8 - len(truth_data['true']) + truth_data['true'] += random.sample( + self.realfacts, missing) + if truth_data['false'] < 8: + missing = 8 - len(truth_data['false']) + truth_data['false'] += random.sample( + self.fakefacts, missing) self.save_user_blob(truth_data, user_data) self.logger.info("Saved truth game data") @@ -234,10 +242,17 @@ def truth_grab(self, user, request): Returns relevant data that the exhibits may want to know """ data = self.load_user_blob(user) - if len(data['true']) > 10: - data['true'] = random.sample(data['true'], 10) - if len(data['false']) > 10: - data['false'] = random.sample(data['false'], 10) + rando = random.randint(0, 1) + if rando == 1: + if len(data['true']) > 7: + data['true'] = random.sample(data['true'], 7) + if len(data['false']) > 8: + data['false'] = random.sample(data['false'], 8) + else: + if len(data['true']) > 8: + data['true'] = random.sample(data['true'], 8) + if len(data['false']) > 7: + data['false'] = random.sample(data['false'], 7) return data @process_api_handler From 6a197b6d7a307779b14ef1bdfc67b1727d3a5b55 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 22:10:20 -0600 Subject: [PATCH 15/30] clear other associations of rfid on association --- lib/rfidb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/rfidb.py b/lib/rfidb.py index 7fad6f6..f575854 100644 --- a/lib/rfidb.py +++ b/lib/rfidb.py @@ -1,6 +1,7 @@ from tornado import gen import rethinkdb as r +from collections import Counter from .dbhelper import RethinkDB from .user import User @@ -53,6 +54,8 @@ def delete_user(self, user_data): def associate_user(self, userid, rfid): conn = yield self.connection() try: + yield r.table('rfid').get_all(rfid, index='rfid') \ + .update({'rfid': None}).run(conn) result = yield r.table('rfid').get(userid).update({'rfid': rfid}) \ .run(conn) if result['skipped'] == 1: From 0b6aa764b28e48fd84494dd3b82856b6e087c37f Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 22:10:38 -0600 Subject: [PATCH 16/30] script to add in actors --- scripts/actors_rfid.py | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 scripts/actors_rfid.py diff --git a/scripts/actors_rfid.py b/scripts/actors_rfid.py new file mode 100644 index 0000000..7b1ffd7 --- /dev/null +++ b/scripts/actors_rfid.py @@ -0,0 +1,52 @@ +import rethinkdb as r +actors = { + "Felicity Turner": "6b8b8d4f-fe8b-4b0e-b257-973b17e343c5", + "Desiree Harlech": "b1f52988-0971-42ab-a515-7c9b0d9f8d9c", + "Amelia Bloom": "1002c758-8005-4b99-ba96-38c0faacffeb", + "Don DeClaire": "67d37fb5-261b-41da-8d3f-2e5aff93ead1", + "Bo Rakenfold": "1423bf50-6bfb-4eb6-997e-25dd1e801a98", + "Lily Jordan": "244f62b1-ee51-444d-84c2-b72a39a2fdbd", +} + + +def create_rfid(name, user_id): + return { + "id": user_id, + "publickey": None, + "privatekey": None, + "showid": None, + "showdate": None, + "name": name, + "rfid": None, + } + + +def create_perm(name, user_id): + return { + 'name': name, + 'id': user_id, + 'delete_processor': False, + 'amelia_processor': True, + 'debug_processor': True, + 'interview_processor': True, + 'mental_health': True, + 'mirror_processor': True, + 'news_processor': True, + 'ownup': True, + 'pr0n_processor': True, + 'recommend_processor': True, + 'romance_processor': True, + 'tos_processor': True, + 'tracked_processor': True, + 'truth_processor': True + } + + +if __name__ == "__main__": + conn = r.connect(db='gulperbase') + for name, user_id in actors.items(): + print("Adding: ", name) + result = r.table("rfid").insert(create_rfid(name, user_id), conflict='update').run(conn) + print(result) + result = r.table("exhibitpermissions").insert(create_perm(name, user_id), conflict='update').run(conn) + print(result) From 4f1c2586c6f1f9c7e1ab40b2b1c1c78e100f2ee7 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 22:11:00 -0600 Subject: [PATCH 17/30] starting to fix truth processor --- lib/processors/truthprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index cc1259e..9f9a91f 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -15,7 +15,7 @@ class TruthProcessor(BaseProcessor): def __init__(self): super().__init__() - self.stopwords = self.load_keywords("stopwords.txt") + self.stopwords = set(self.load_keywords("stopwords.txt")) self.fakefacts = self.load_keywords("fakefacts.txt") self.realfacts = self.load_keywords("realfacts.txt") self.truths = random.randint(7, 8) @@ -25,7 +25,7 @@ def get_words(self, text_list): for text in text_list: if text is None: continue - words.extend(re.findall(r"[\w']+", text)) + words.extend(re.findall(r"\b[\w']+\b", text)) lower = [word.lower() for word in words] return lower From df128a06a496ba70181eb7691ef4798db5e59328 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Wed, 27 Apr 2016 23:42:50 -0600 Subject: [PATCH 18/30] speedups to truth --- lib/processors/__init__.py | 2 +- lib/processors/truthprocessor.py | 56 +++++++++++++------------------- 2 files changed, 23 insertions(+), 35 deletions(-) diff --git a/lib/processors/__init__.py b/lib/processors/__init__.py index f956735..956a9ce 100644 --- a/lib/processors/__init__.py +++ b/lib/processors/__init__.py @@ -21,8 +21,8 @@ processors = [ DebugProcessor(), - Pr0nProcessor(), TruthProcessor(), + Pr0nProcessor(), MirrorProcessor(), NewsProcessor(), OwnupProcessor(), diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index 9f9a91f..4b73ab3 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -21,28 +21,14 @@ def __init__(self): self.truths = random.randint(7, 8) def get_words(self, text_list): - words = [] for text in text_list: - if text is None: - continue - words.extend(re.findall(r"\b[\w']+\b", text)) - lower = [word.lower() for word in words] - return lower + if text: + words = re.findall(r"\b[\w']+\b", text.lower()) + yield from words def word_freq(self, wordlist): cleanwords = [w for w in wordlist if w not in self.stopwords] - wordfreq = [wordlist.count(p) for p in cleanwords] - freqdict = dict(zip(cleanwords, wordfreq)) - aux = [(freqdict[key], key) for key in freqdict] - aux.sort() - aux.reverse() - return aux - - def get_num_uses(self, word, wordfreq): - for wordf in wordfreq: - if wordf[1] == word: - return wordf[0] - return 0 + return Counter(cleanwords) def get_percentage(self, word, text_list): total = float(len(text_list)) @@ -84,8 +70,8 @@ def _common_and_lie(self, items): lie, _ = random.choice(top_10[1:]) return best_item, lie - def percentage_check(self, word, text_list, thresh, fact_str, facts, lies): - freq = self.get_percentage(word, text_list) + def percentage_check(self, word, word_counts, num_messages, thresh, fact_str, facts, lies): + freq = word_counts[word] / float(num_messages) if freq > thresh: if random.randint(0, 1) == 0 and len(facts) <= 5: facts.append( @@ -99,7 +85,7 @@ def percentage_check(self, word, text_list, thresh, fact_str, facts, lies): return facts, lies def check_uses(self, word, freq, thresh, fact_str, facts, lies, which): - quant = self.get_num_uses(word, freq) + quant = freq[word] if which == 0: if quant >= thresh and len(facts) <= self.truths: facts.append( @@ -157,30 +143,31 @@ def process(self, user_data): if user_data.data['gmail'].get('text'): gwords = self.get_words(user_data.data['gmail']['text']) gfreq = self.word_freq(gwords) - if len(gfreq) > 0: + if gfreq: if random.randint(0, 1) == 0: + most_common = gfreq.most_common(1) truth_data['true'].append( "Besides articles, prepositions, and pronouns your most " - "common word in email is {0}".format(gfreq[0][1])) + "common word in email is {0}".format(most_common[0][0])) else: - word_len = len(gfreq) - grab = round(word_len * .5) + fake = random.choice(list(gfreq.keys())) truth_data['false'].append( "Besides articles, prepositions, and pronouns your most " - "common word in email is \"{0}\"".format(gfreq[grab][1])) + "common word in email is \"{0}\"".format(fake)) - if user_data.data.get('fbtext'): - text_list = [post['text'] - for post in user_data.data['fbtext']['text']] - fbwords = self.get_words(text_list) + if user_data.data.get('fbtext') and user_data.data['fbtext']['text']: + text_list = (post['text'] + for post in user_data.data['fbtext']['text']) + N = len(user_data.data['fbtext']['text']) + fbwords = set(self.get_words(text_list)) fbfreq = self.word_freq(fbwords) # Later this can be a loop that tried different word, token pairs truth_data['true'], truth_data['false'] = self.percentage_check( - 'me', text_list, .1, + 'me', fbfreq, N, .1, "You use the word \"me\" in {0}% of your facebook posts", truth_data['true'], truth_data['false']) truth_data['true'], truth_data['false'] = self.percentage_check( - 'fuck', text_list, .05, + 'fuck', fbfreq, N, .05, "You use the word \"fuck\" in {0}% of your facebook posts", truth_data['true'], truth_data['false']) # Now going to get into checking number of word uses @@ -220,11 +207,12 @@ def process(self, user_data): random.shuffle(truth_data['false']) truth_data['true'] = truth_data['true'][:self.num_items] truth_data['false'] = truth_data['false'][:self.num_items] - if truth_data['true'] < 8: + self.logger.info("True: %d, False: %d", len(truth_data['true']), len(truth_data['false'])) + if len(truth_data['true']) < 8: missing = 8 - len(truth_data['true']) truth_data['true'] += random.sample( self.realfacts, missing) - if truth_data['false'] < 8: + if len(truth_data['false']) < 8: missing = 8 - len(truth_data['false']) truth_data['false'] += random.sample( self.fakefacts, missing) From 3ba8d00bcc7a64c015eebbcccc62c771997e753c Mon Sep 17 00:00:00 2001 From: wannabeCitizen Date: Thu, 28 Apr 2016 00:35:39 -0600 Subject: [PATCH 19/30] added a few words to our lists, and made (hopefully a much better truth processor --- lib/processors/data/mentalwords.txt | 2 + lib/processors/data/partywords.txt | 3 +- lib/processors/data/positiveworkwords.txt | 3 +- lib/processors/truthprocessor.py | 122 ++++++++++++---------- 4 files changed, 72 insertions(+), 58 deletions(-) diff --git a/lib/processors/data/mentalwords.txt b/lib/processors/data/mentalwords.txt index 87b317e..03ad48d 100644 --- a/lib/processors/data/mentalwords.txt +++ b/lib/processors/data/mentalwords.txt @@ -30,6 +30,8 @@ worse upsetting easy + best + worst boredom confusion confused diff --git a/lib/processors/data/partywords.txt b/lib/processors/data/partywords.txt index 62e730a..1b92ec0 100644 --- a/lib/processors/data/partywords.txt +++ b/lib/processors/data/partywords.txt @@ -37,4 +37,5 @@ trees marijuana cannibas stoned -420 \ No newline at end of file +420 +dafuq \ No newline at end of file diff --git a/lib/processors/data/positiveworkwords.txt b/lib/processors/data/positiveworkwords.txt index dcfb3a2..64b63da 100644 --- a/lib/processors/data/positiveworkwords.txt +++ b/lib/processors/data/positiveworkwords.txt @@ -12,4 +12,5 @@ still at work love my boss best boss finished my homework -allnighter \ No newline at end of file +allnighter +working \ No newline at end of file diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index 4b73ab3..37fd994 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -18,7 +18,10 @@ def __init__(self): self.stopwords = set(self.load_keywords("stopwords.txt")) self.fakefacts = self.load_keywords("fakefacts.txt") self.realfacts = self.load_keywords("realfacts.txt") - self.truths = random.randint(7, 8) + self.emowords = self.load_keywords("mentalwords.text") + self.partywords = self.load_keywords("partywords.txt") + self.healthwords = self.load_keywords("healthwords.txt") + self.poswork = self.load_keywords("positiveworkwords.txt") def get_words(self, text_list): for text in text_list: @@ -70,32 +73,47 @@ def _common_and_lie(self, items): lie, _ = random.choice(top_10[1:]) return best_item, lie - def percentage_check(self, word, word_counts, num_messages, thresh, fact_str, facts, lies): - freq = word_counts[word] / float(num_messages) - if freq > thresh: - if random.randint(0, 1) == 0 and len(facts) <= 5: - facts.append( - fact_str.format(round(freq * 100))) - else: - rand = 0 - while rand == 0: - rand = random.randint(-(thresh*10), thresh*10) - lies.append( - fact_str.format(round(freq * 100 + rand))) - return facts, lies - - def check_uses(self, word, freq, thresh, fact_str, facts, lies, which): - quant = freq[word] - if which == 0: - if quant >= thresh and len(facts) <= self.truths: - facts.append( - fact_str.format(quant)) - else: - if quant >= thresh and len(lies) <= 15 - self.truths: - lies.append( - fact_str.format(quant * random.randint(round(thresh/3), - thresh*3))) - return facts, lies + def percentage_check(self, freqmap, N): + str_build = "You use the word '{0}' in approximately {1}% of your facebook posts" + new_facts = [] + new_lies = [] + words = self.partywords + self.emowords + for word in words: + if word in freqmap: + freq = freqmap[word] / float(N) + if freq < .05: + continue + if random.randint(0, 1) == 0: + new_facts.append( + str_build(word, round(freq * 100))) + else: + rand = 0 + while rand == 0: + rand = random.randint(-(freq*80), freq*200) + new_lies.append( + str_build.format(word, round(freq * 100 + rand))) + return new_facts, new_lies + + def check_uses(self, freq): + str_build = "You use the word '{0}' at least {1}% on Facebook" + new_facts = [] + new_lies = [] + words = self.partywords + self.emowords + self.healthwords + self.poswork + for word in words: + if word in freqmap: + freq = freqmap[word] + if freq < 5: + continue + if random.randint(0, 1) == 0: + new_facts.append( + str_build.format(word, freq)) + else: + rand = 0 + while rand == 0: + rand = random.randint(-round(freq*.8), freq*3) + new_lies.append( + str_build.format(word, freq + rand)) + return new_facts, new_lies @gen.coroutine def process(self, user_data): @@ -159,40 +177,34 @@ def process(self, user_data): text_list = (post['text'] for post in user_data.data['fbtext']['text']) N = len(user_data.data['fbtext']['text']) - fbwords = set(self.get_words(text_list)) + fbwords = self.get_words(text_list) fbfreq = self.word_freq(fbwords) # Later this can be a loop that tried different word, token pairs - truth_data['true'], truth_data['false'] = self.percentage_check( - 'me', fbfreq, N, .1, - "You use the word \"me\" in {0}% of your facebook posts", - truth_data['true'], truth_data['false']) - truth_data['true'], truth_data['false'] = self.percentage_check( - 'fuck', fbfreq, N, .05, - "You use the word \"fuck\" in {0}% of your facebook posts", - truth_data['true'], truth_data['false']) - # Now going to get into checking number of word uses - truth_data['true'], truth_data['false'] = self.check_uses( - 'yass', fbfreq, 5, - "You used the word \"yass\" {0} times on facebook", - truth_data['true'], truth_data['false'], 0) - truth_data['true'], truth_data['false'] = self.check_uses( - 'dafuq', fbfreq, 5, - "You used the word \"dafuq\" {0} times on facebook", - truth_data['true'], truth_data['false'], 0) - truth_data['true'], truth_data['false'] = self.check_uses( - 'lol', fbfreq, 5, - "You used the word \"LOL\" {0} times on facebook", - truth_data['true'], truth_data['false'], 1) - truth_data['true'], truth_data['false'] = self.check_uses( - 'love', fbfreq, 5, - "You used the word \"love\" {0} times on facebook", - truth_data['true'], truth_data['false'], 0) + if fbfreq: + if random.randint(0, 1) == 0: + most_common = fbfreq.most_common(1) + truth_data['true'].append( + "Besides articles, prepositions, and pronouns your most " + "common word on Facebook is {0}".format(most_common[0][0])) + else: + fake = random.choice(list(fbfreq.keys())) + truth_data['true'].append( + "Besides articles, prepositions, and pronouns your most " + "common word on Facebook is {0}".format(fake)) + + new_facts, new_lies = self.percentage_check(fbfreq, N) + truth_data['true'] += new_facts + truth_data['false'] += new_lies + new_facts, new_lies = self.check_uses(fbfreq) + truth_data['true'] += new_facts + truth_data['false'] += new_lies + if user_data.data.get('reddit'): try: fact, lie = self.common_subreddit( user_data.data['reddit']['submissions']) - if (len(truth_data['true']) < len(truth_data['false'])): + if (len(truth_data['true']) <= len(truth_data['false'])): truth_data['true'].append( "Your most common subreddit you submit to is {0}".format( fact)) @@ -205,8 +217,6 @@ def process(self, user_data): random.shuffle(truth_data['true']) random.shuffle(truth_data['false']) - truth_data['true'] = truth_data['true'][:self.num_items] - truth_data['false'] = truth_data['false'][:self.num_items] self.logger.info("True: %d, False: %d", len(truth_data['true']), len(truth_data['false'])) if len(truth_data['true']) < 8: missing = 8 - len(truth_data['true']) From dbcc2413591d2ef2110b9c1ad9cc614613de5a9c Mon Sep 17 00:00:00 2001 From: wannabeCitizen Date: Thu, 28 Apr 2016 00:36:24 -0600 Subject: [PATCH 20/30] typo fix --- lib/processors/truthprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index 37fd994..03dbadf 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -94,7 +94,7 @@ def percentage_check(self, freqmap, N): str_build.format(word, round(freq * 100 + rand))) return new_facts, new_lies - def check_uses(self, freq): + def check_uses(self, freqmap): str_build = "You use the word '{0}' at least {1}% on Facebook" new_facts = [] new_lies = [] From d9e8b6f5a8723463995855019f87e7b82b9bcfaf Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Thu, 28 Apr 2016 11:00:47 -0600 Subject: [PATCH 21/30] added h4x0r --- scripts/actors_rfid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/actors_rfid.py b/scripts/actors_rfid.py index 7b1ffd7..c9387e2 100644 --- a/scripts/actors_rfid.py +++ b/scripts/actors_rfid.py @@ -6,6 +6,7 @@ "Don DeClaire": "67d37fb5-261b-41da-8d3f-2e5aff93ead1", "Bo Rakenfold": "1423bf50-6bfb-4eb6-997e-25dd1e801a98", "Lily Jordan": "244f62b1-ee51-444d-84c2-b72a39a2fdbd", + "hacker": "hacker", } From bc2e2c831c6e4741aa9d2866d93693f37a678abf Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Thu, 28 Apr 2016 13:18:31 -0600 Subject: [PATCH 22/30] typo --- lib/processors/truthprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index 03dbadf..ddc1997 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -18,7 +18,7 @@ def __init__(self): self.stopwords = set(self.load_keywords("stopwords.txt")) self.fakefacts = self.load_keywords("fakefacts.txt") self.realfacts = self.load_keywords("realfacts.txt") - self.emowords = self.load_keywords("mentalwords.text") + self.emowords = self.load_keywords("mentalwords.txt") self.partywords = self.load_keywords("partywords.txt") self.healthwords = self.load_keywords("healthwords.txt") self.poswork = self.load_keywords("positiveworkwords.txt") From 7269cd6fb0c31b037f5e853156d242e04580f860 Mon Sep 17 00:00:00 2001 From: wannabeCitizen Date: Thu, 28 Apr 2016 14:37:06 -0600 Subject: [PATCH 23/30] fixed truth error --- lib/processors/truthprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index 03dbadf..e508b9f 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -85,11 +85,11 @@ def percentage_check(self, freqmap, N): continue if random.randint(0, 1) == 0: new_facts.append( - str_build(word, round(freq * 100))) + str_build.format(word, round(freq * 100))) else: rand = 0 while rand == 0: - rand = random.randint(-(freq*80), freq*200) + rand = random.randint(-int(freq*80), int(freq*200)) new_lies.append( str_build.format(word, round(freq * 100 + rand))) return new_facts, new_lies From da26b517e2a154e3c1075325a3073c01eaa44363 Mon Sep 17 00:00:00 2001 From: wannabeCitizen Date: Thu, 28 Apr 2016 14:52:45 -0600 Subject: [PATCH 24/30] fixed typo --- lib/processors/truthprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index 272b223..d88d76c 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -95,7 +95,7 @@ def percentage_check(self, freqmap, N): return new_facts, new_lies def check_uses(self, freqmap): - str_build = "You use the word '{0}' at least {1}% on Facebook" + str_build = "You used the word '{0}' at least {1} times on Facebook" new_facts = [] new_lies = [] words = self.partywords + self.emowords + self.healthwords + self.poswork From f3f38e3aee5610ce1e2ac0b957f6e134f9267313 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Thu, 28 Apr 2016 18:09:36 -0600 Subject: [PATCH 25/30] add email --- scripts/actors_rfid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/actors_rfid.py b/scripts/actors_rfid.py index c9387e2..bfceaee 100644 --- a/scripts/actors_rfid.py +++ b/scripts/actors_rfid.py @@ -7,6 +7,7 @@ "Bo Rakenfold": "1423bf50-6bfb-4eb6-997e-25dd1e801a98", "Lily Jordan": "244f62b1-ee51-444d-84c2-b72a39a2fdbd", "hacker": "hacker", + "email": "email", } From dd414f9487d90a53d8b7d05aa1176b34dfc67a03 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Thu, 28 Apr 2016 18:13:13 -0600 Subject: [PATCH 26/30] deleted all rfids accidentally --- lib/rfidb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rfidb.py b/lib/rfidb.py index f575854..9a72e23 100644 --- a/lib/rfidb.py +++ b/lib/rfidb.py @@ -54,7 +54,7 @@ def delete_user(self, user_data): def associate_user(self, userid, rfid): conn = yield self.connection() try: - yield r.table('rfid').get_all(rfid, index='rfid') \ + yield r.table('rfid').filter({"rfid": rfid}) \ .update({'rfid': None}).run(conn) result = yield r.table('rfid').get(userid).update({'rfid': rfid}) \ .run(conn) From cd543f1f78c955f96bbc95ccf5f0b4e8c177be77 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Fri, 29 Apr 2016 01:46:05 -0600 Subject: [PATCH 27/30] added luke's error --- lib/processors/pr0nprocessor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/processors/pr0nprocessor.py b/lib/processors/pr0nprocessor.py index dfd3f2d..3c71ad8 100644 --- a/lib/processors/pr0nprocessor.py +++ b/lib/processors/pr0nprocessor.py @@ -199,7 +199,10 @@ def set_preference(self, user, request): Sets a users preference for a given photo """ image_id = request.get_argument("id") - preference = int(request.get_argument("preference")) + try: + preference = int(request.get_argument("preference")) + except ValueError: # luke's error + preference = 0 data = self.load_user_blob(user) images_data = data['images_to_scores'] names_data = data['names_to_scores'] From 86110b90605e3f7aa53532a1585253bf5092337c Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Fri, 29 Apr 2016 15:46:26 -0600 Subject: [PATCH 28/30] more logging for face hashing --- lib/processors/pr0nprocessor.py | 2 +- lib/scrapers/fbphotos.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/processors/pr0nprocessor.py b/lib/processors/pr0nprocessor.py index 3c71ad8..146bf5f 100644 --- a/lib/processors/pr0nprocessor.py +++ b/lib/processors/pr0nprocessor.py @@ -201,7 +201,7 @@ def set_preference(self, user, request): image_id = request.get_argument("id") try: preference = int(request.get_argument("preference")) - except ValueError: # luke's error + except ValueError: # luke's error preference = 0 data = self.load_user_blob(user) images_data = data['images_to_scores'] diff --git a/lib/scrapers/fbphotos.py b/lib/scrapers/fbphotos.py index caae80e..a22487a 100644 --- a/lib/scrapers/fbphotos.py +++ b/lib/scrapers/fbphotos.py @@ -81,12 +81,13 @@ def get_friends_profile(self, graph): for d in friends_raw} pictures = [] profile_pic = 'http://graph.facebook.com/{}/picture?type=large' - for fid, fname in friends.items(): + for i, (fid, fname) in enumerate(friends.items()): photo = { 'url': profile_pic.format(fid), 'id': 'img-' + fid, } + self.logger.debug("Friend face finding: %d / %d", i, len(friends)) faces = yield find_faces_url(photo['url'], hash_face=True) # go through the faces _we_ found and interpolate those results # with the tags from the image @@ -98,7 +99,7 @@ def get_friends_profile(self, graph): @gen.coroutine def parse_photos(self, graph, photos): - for photo in photos: + for i, photo in enumerate(photos): if 'tags' not in photo: continue photo['images'].sort(key=lambda x: x['height']*x['width']) @@ -113,6 +114,7 @@ def parse_photos(self, graph, photos): for t in photo['tags']['data'] if t.get('x') and t.get('y') ] + self.logger.debug("Face finding: %d / %d", i, len(photos)) faces = yield find_faces_url(image_url['source'], hash_face=True) # go through the faces _we_ found and interpolate those results # with the tags from the image From 1bf03f28022e1240da506028b2553a4f2394d649 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Sat, 30 Apr 2016 16:19:40 -0600 Subject: [PATCH 29/30] reduced number of fb photos --- lib/scrapers/fbphotos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scrapers/fbphotos.py b/lib/scrapers/fbphotos.py index a22487a..4068047 100644 --- a/lib/scrapers/fbphotos.py +++ b/lib/scrapers/fbphotos.py @@ -15,7 +15,7 @@ class FBPhotosScraper(BaseScraper): @property def num_images_per_user(self): if CONFIG['_mode'] == 'prod': - return 300 + return 200 return 50 @gen.coroutine From 037de6baf827b872b751ece141c5041014cbe4d7 Mon Sep 17 00:00:00 2001 From: Micha Gorelick Date: Sat, 30 Apr 2016 16:20:53 -0600 Subject: [PATCH 30/30] luke's error v2 --- lib/processors/pr0nprocessor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/processors/pr0nprocessor.py b/lib/processors/pr0nprocessor.py index 146bf5f..ed69417 100644 --- a/lib/processors/pr0nprocessor.py +++ b/lib/processors/pr0nprocessor.py @@ -208,7 +208,10 @@ def set_preference(self, user, request): names_data = data['names_to_scores'] # increase the direct preference - images_data[image_id]['scores']['direct'] += preference + try: + images_data[image_id]['scores']['direct'] += preference + except KeyError: # luke's error + return None # increase all images that have the same person in them for name, dist in images_data[image_id]['names'].items(): dist = float(dist)