diff --git a/app/user.py b/app/user.py index c3b9085..8e7aff3 100644 --- a/app/user.py +++ b/app/user.py @@ -25,7 +25,7 @@ def post(self): meta = data.get('meta', None) user = User(userid, publickey_pem, privatekey_pem=privatekey_pem, services=data['services'], meta=meta) - ioloop.IOLoop.current().add_callback(partial(userprocess, user)) + ioloop.IOLoop.current().add_callback(partial(userprocess, [user])) self.api_response("OK") @@ -37,6 +37,7 @@ def get(self): shares = self.get_arguments('share') passphrase = self.get_argument('passphrase', None) reprocess = bool(self.get_argument('reprocess', None) is not None) + scrape_cache = not bool(self.get_argument('scrape_cache', None) is not None) ticket_api = CONFIG.get('ticket_api') # we could also just pass the raw arguments, but this is more explicit @@ -54,6 +55,7 @@ def get(self): show_data = json.loads(show_data_raw.body) show_date = show_data['data']['date'] users_added = [] + users_to_process = [] for user_data in show_data['data']['users']: userid = user_data.pop('id') perms = yield exhibitperms.get_permissions(userid) @@ -61,6 +63,7 @@ def get(self): users_added.append({'userid': userid, 'permissions': perms, 'process': False}) + continue publickey = user_data['publickey'] privatekey = user_data.get('privatekey') meta = user_data.get('meta') or {} @@ -69,7 +72,10 @@ def get(self): user = User(userid, publickey, services=user_data['services'], privatekey_pem=privatekey, meta=meta) users_added.append({'userid': userid, 'process': True}) - ioloop.IOLoop.current().add_callback(partial(userprocess, user)) + users_to_process.append(user) + ioloop.IOLoop.current().spawn_callback( + partial(userprocess, users_to_process, scrape_cache) + ) return self.api_response(users_added) diff --git a/lib/processors/__init__.py b/lib/processors/__init__.py index c7b07cb..956a9ce 100644 --- a/lib/processors/__init__.py +++ b/lib/processors/__init__.py @@ -20,8 +20,9 @@ logger = logging.getLogger("processor.process") processors = [ - Pr0nProcessor(), + DebugProcessor(), TruthProcessor(), + Pr0nProcessor(), MirrorProcessor(), NewsProcessor(), OwnupProcessor(), @@ -33,7 +34,6 @@ RomanceProcessor(), TOSProcessor(), DeleteProcessor(), - DebugProcessor(), ] diff --git a/lib/processors/ameliaprocessor.py b/lib/processors/ameliaprocessor.py index c2edcdc..cec3307 100644 --- a/lib/processors/ameliaprocessor.py +++ b/lib/processors/ameliaprocessor.py @@ -9,6 +9,7 @@ class AmeliaProcessor(BaseProcessor): name = 'amelia_processor' + auth = False def __init__(self): super().__init__() diff --git a/lib/processors/data/mentalwords.txt b/lib/processors/data/mentalwords.txt index 87b317e..03ad48d 100644 --- a/lib/processors/data/mentalwords.txt +++ b/lib/processors/data/mentalwords.txt @@ -30,6 +30,8 @@ worse upsetting easy + best + worst boredom confusion confused diff --git a/lib/processors/data/partywords.txt b/lib/processors/data/partywords.txt index 62e730a..1b92ec0 100644 --- a/lib/processors/data/partywords.txt +++ b/lib/processors/data/partywords.txt @@ -37,4 +37,5 @@ trees marijuana cannibas stoned -420 \ No newline at end of file +420 +dafuq \ No newline at end of file diff --git a/lib/processors/data/positiveworkwords.txt b/lib/processors/data/positiveworkwords.txt index dcfb3a2..64b63da 100644 --- a/lib/processors/data/positiveworkwords.txt +++ b/lib/processors/data/positiveworkwords.txt @@ -12,4 +12,5 @@ still at work love my boss best boss finished my homework -allnighter \ No newline at end of file +allnighter +working \ No newline at end of file diff --git a/lib/processors/debugprocessor.py b/lib/processors/debugprocessor.py index 71e1f78..71ed514 100644 --- a/lib/processors/debugprocessor.py +++ b/lib/processors/debugprocessor.py @@ -1,8 +1,5 @@ from tornado import gen -import pickle -import os -from ..config import CONFIG from .lib.baseprocessor import BaseProcessor @@ -14,12 +11,10 @@ def process(self, user_data): """ Save user data for inspection """ - if CONFIG.get('_mode') != 'dev': - return False - filename = "./data/debug/{}.pkl".format(user_data.userid) - os.makedirs('./data/debug', exist_ok=True) - with open(filename, 'wb+') as fd: - pickle.dump(user_data.data, fd) - self.logger.info("Saved user {} to {}".format(user_data.userid, - filename)) + self.logger.info("Saving user scrape data") + self.save_user_blob(user_data.data, user_data) + self.logger.info("Saved user: " + user_data.userid) return True + + def load_scrape(self, user_data): + return self.load_user_blob(user_data) diff --git a/lib/processors/interviewprocessor.py b/lib/processors/interviewprocessor.py index 31a549e..8514086 100644 --- a/lib/processors/interviewprocessor.py +++ b/lib/processors/interviewprocessor.py @@ -175,8 +175,9 @@ def process(self, user_data): pos_interests = self.scan_pos_interests( pos_interests, likes) - testit = pos_quotes + neg_quotes + pos_interests + neg_interests + pos_events + neg_events - if len(testit) <= 2: + num_quotes = sum(len(quotes) for quotes in (pos_quotes, neg_quotes, + pos_interests, neg_interests, pos_events, neg_events)) + if num_quotes <= 2: return False interview_data = self.build_data(interview_data, pos_quotes, neg_quotes, diff --git a/lib/processors/lib/baseprocessor.py b/lib/processors/lib/baseprocessor.py index ea39025..c2148b9 100644 --- a/lib/processors/lib/baseprocessor.py +++ b/lib/processors/lib/baseprocessor.py @@ -72,7 +72,7 @@ def register_handlers(self): def save_user_blob(self, blob, user): filedata = dict(name=self.name, uid=user.userid) - os.makedirs("./data/{name}/user".format(**filedata), exist_ok=True) + os.makedirs("./data/{name}/user/".format(**filedata), exist_ok=True) if CONFIG.get('_mode') == 'dev': filename = "./data/{name}/user/{uid}.pkl".format(**filedata) with open(filename, 'wb+') as fd: diff --git a/lib/processors/lib/keyword_filters.py b/lib/processors/lib/keyword_filters.py new file mode 100644 index 0000000..8c74a2e --- /dev/null +++ b/lib/processors/lib/keyword_filters.py @@ -0,0 +1,57 @@ +import re + + +def is_great_quote(text, keywords): + text = text.lower() + words = set(text.split()) + if len(words) < 3: + return False + if any(len(w) > 13 or 'http' in w + for w in words): + return False + if "@" in text: + return False + for keyword in keywords: + if keyword in words: + return True + return False + + +def process_post(text, keywords): + # If there is enough perm quotes, quit. + sentences = re.split(r' *[\.\?!][\'"\)\]]* *', text) + for sentence in map(str.strip, sentences): + if is_great_quote(sentence, keywords): + yield sentence + + +def process_facebook(user_data, keywords): + if user_data.data.get('fbtext', None): + fbtext = user_data.data['fbtext'] + fbposts = fbtext.get('text') or [] + for post in fbposts: + yield from process_post(post['text'], keywords) + + +def process_twitter(user_data, keywords): + if user_data.data.get('twitter', None): + twitter = user_data.data['twitter'] + tweets = twitter.get('tweets') or [] + for post in tweets: + yield from process_post(post, keywords) + + +def process_reddit(user_data, keywords): + if user_data.data.get('reddit', None): + reddit = user_data.data['reddit'] + posts = reddit.get('text') or [] + for post in posts: + yield from process_post(post['body'], keywords) + + +def process_gmail(user_data, keywords): + if user_data.data.get('gmail'): + gmail = user_data.data['gmail'] + snippets = gmail.get('snippets') or [] + for post in snippets: + yield from process_post(post, keywords) diff --git a/lib/processors/mentalhealthprocessor.py b/lib/processors/mentalhealthprocessor.py index 7b55c50..9d0586d 100644 --- a/lib/processors/mentalhealthprocessor.py +++ b/lib/processors/mentalhealthprocessor.py @@ -1,64 +1,38 @@ from tornado import gen -import itertools as IT + +import random from .lib.handler import process_api_handler from .lib.baseprocessor import BaseProcessor +from .lib import keyword_filters class MentalHealthProcessor(BaseProcessor): name = 'mental_health' - limit = 200 + num_quotes = 200 def __init__(self): super().__init__() self.keywords = self.load_keywords('mentalwords.txt') - def is_good_quote(self, text): - text = text.lower() - for word in self.keywords: - if word in text: - return True - return False - - def process_facebook(self, user_data): - if user_data.data.get('fbtext'): - fbtext = user_data.data['fbtext'] - fbposts = fbtext.get('text', []) - for post in fbposts: - yield post['text'] - - def process_twitter(self, user_data): - if user_data.data.get('twitter'): - twitter = user_data.data['twitter'] - tweets = twitter.get('tweets', []) - for post in tweets: - yield post - - def process_reddit(self, user_data): - if user_data.data.get('reddit'): - reddit = user_data.data['reddit'] - posts = reddit.get('text', []) - for post in posts: - yield post['body'] - - def process_gmail(self, user_data): - if user_data.data.get('gmail'): - gmail = user_data.data['gmail'] - for post in gmail.get('snippet', []): - yield post - @gen.coroutine def process(self, user_data): self.logger.info("Processing user: {}".format(user_data.userid)) - quote_candidates = IT.chain(self.process_facebook(user_data), - self.process_twitter(user_data), - self.process_reddit(user_data), - self.process_gmail(user_data)) - quotes = list(IT.islice(quote_candidates, self.limit)) + quotes = [] + quotes += list(keyword_filters.process_facebook(user_data, + self.keywords)) + quotes += list(keyword_filters.process_twitter(user_data, + self.keywords)) + quotes += list(keyword_filters.process_reddit(user_data, + self.keywords)) + quotes += list(keyword_filters.process_gmail(user_data, + self.keywords)) self.logger.debug("User {}: {} quotes".format(user_data.userid, len(quotes))) - if len(quotes) < 0: + if len(quotes) < 5: return False + if len(quotes) > self.num_quotes: + quotes = random.sample(quotes, self.num_quotes) self.save_user_blob(quotes, user_data) return True diff --git a/lib/processors/ownupprocessor.py b/lib/processors/ownupprocessor.py index 748bd9a..e81e69e 100644 --- a/lib/processors/ownupprocessor.py +++ b/lib/processors/ownupprocessor.py @@ -2,90 +2,45 @@ from .lib.handler import process_api_handler from .lib.baseprocessor import BaseProcessor +from .lib import keyword_filters import random class OwnupProcessor(BaseProcessor): name = 'ownup' - limit = 10 + num_quotes = 100 def __init__(self): super().__init__() self.keywords = self.load_keywords('ownup.txt') - def is_great_quote(self, text): - text = text.lower() - for word in self.keywords: - if text.find(word) >= 0: - return True - return False - - """ Return True if processing should continue - """ - def process_post(self, text, temp, perm): - # If there is enough perm quotes, quit. - if len(perm) >= self.limit: - return False - sentences = text.split('.') - for sentence in sentences: - if self.is_great_quote(sentence): - perm.append(sentence) - random.shuffle(perm) - elif len(sentence) > 15: - temp.append(sentence) - random.shuffle(perm) - return True - - def process_facebook(self, user_data, temp, perm): - if user_data.data.get('fbtext', None): - fbtext = user_data.data.get('fbtext') - if not fbtext or len(perm) >= self.limit: - return - fbposts = fbtext['text'] - for post in fbposts: - if not self.process_post(post['text'], temp, perm): - return - - def process_twitter(self, user_data, temp, perm): - if user_data.data.get('twitter', None): - twitter = user_data.data.get('twitter', None) - if not twitter or len(perm) >= self.limit: - return - tweets = twitter['tweets'] - for post in tweets: - if not self.process_post(post, temp, perm): - return - - def process_reddit(self, user_data, temp, perm): - if user_data.data.get('reddit', None): - reddit = user_data.data.get('reddit', None) - if not reddit or len(perm) >= self.limit: - return - posts = reddit['text'] - for post in posts: - if not self.process_post(post['body'], temp, perm): - return - @gen.coroutine def process(self, user_data): - self.logger.info("[OU] Processing user: {}".format(user_data.userid)) - temp = [] - perm = [] - self.process_facebook(user_data, temp, perm) - self.process_twitter(user_data, temp, perm) - self.process_reddit(user_data, temp, perm) - while len(perm) < self.limit and len(temp) > 1: - perm.append(temp.pop()) - if len(perm) < 3: + self.logger.info("Processing user: {}".format(user_data.userid)) + quotes = [] + quotes += list(keyword_filters.process_facebook(user_data, + self.keywords)) + quotes += list(keyword_filters.process_twitter(user_data, + self.keywords)) + quotes += list(keyword_filters.process_reddit(user_data, + self.keywords)) + self.logger.debug("User %s has %d quotes", user_data.userid, len(quotes)) + if not quotes: return False - blob = {'name': user_data.meta['name'], 'quotes': perm} + if len(quotes) > self.num_quotes: + quotes = random.sample(quotes, self.num_quotes) + blob = {'name': user_data.meta['name'], + 'quotes': quotes} self.save_user_blob(blob, user_data) return True @gen.coroutine def get_quotes(self, user, request): - return self.load_user_blob(user) + data = self.load_user_blob(user) + if len(data['quotes']) > 10: + data['quotes'] = random.sample(data['quotes'], 10) + return data @process_api_handler def register_handlers(self): diff --git a/lib/processors/pr0nprocessor.py b/lib/processors/pr0nprocessor.py index dfd3f2d..ed69417 100644 --- a/lib/processors/pr0nprocessor.py +++ b/lib/processors/pr0nprocessor.py @@ -199,13 +199,19 @@ def set_preference(self, user, request): Sets a users preference for a given photo """ image_id = request.get_argument("id") - preference = int(request.get_argument("preference")) + try: + preference = int(request.get_argument("preference")) + except ValueError: # luke's error + preference = 0 data = self.load_user_blob(user) images_data = data['images_to_scores'] names_data = data['names_to_scores'] # increase the direct preference - images_data[image_id]['scores']['direct'] += preference + try: + images_data[image_id]['scores']['direct'] += preference + except KeyError: # luke's error + return None # increase all images that have the same person in them for name, dist in images_data[image_id]['names'].items(): dist = float(dist) diff --git a/lib/processors/truthprocessor.py b/lib/processors/truthprocessor.py index 30e7720..d88d76c 100644 --- a/lib/processors/truthprocessor.py +++ b/lib/processors/truthprocessor.py @@ -11,37 +11,27 @@ class TruthProcessor(BaseProcessor): name = 'truth_processor' data = {} + num_items = 100 def __init__(self): super().__init__() - self.stopwords = self.load_keywords("stopwords.txt") + self.stopwords = set(self.load_keywords("stopwords.txt")) self.fakefacts = self.load_keywords("fakefacts.txt") self.realfacts = self.load_keywords("realfacts.txt") - self.truths = random.randint(7, 8) + self.emowords = self.load_keywords("mentalwords.txt") + self.partywords = self.load_keywords("partywords.txt") + self.healthwords = self.load_keywords("healthwords.txt") + self.poswork = self.load_keywords("positiveworkwords.txt") def get_words(self, text_list): - words = [] for text in text_list: - if text is None: - continue - words.extend(re.findall(r"[\w']+", text)) - lower = [word.lower() for word in words] - return lower + if text: + words = re.findall(r"\b[\w']+\b", text.lower()) + yield from words def word_freq(self, wordlist): cleanwords = [w for w in wordlist if w not in self.stopwords] - wordfreq = [wordlist.count(p) for p in cleanwords] - freqdict = dict(zip(cleanwords, wordfreq)) - aux = [(freqdict[key], key) for key in freqdict] - aux.sort() - aux.reverse() - return aux - - def get_num_uses(self, word, wordfreq): - for wordf in wordfreq: - if wordf[1] == word: - return wordf[0] - return 0 + return Counter(cleanwords) def get_percentage(self, word, text_list): total = float(len(text_list)) @@ -83,42 +73,47 @@ def _common_and_lie(self, items): lie, _ = random.choice(top_10[1:]) return best_item, lie - def fill_truths(self, truestuff): - while len(truestuff) < self.truths: - truestuff.append(random.choice(self.realfacts)) - return truestuff - - def fill_lies(self, lies): - while len(lies) < (15 - self.truths): - lies.append(random.choice(self.fakefacts)) - return lies - - def percentage_check(self, word, text_list, thresh, fact_str, facts, lies): - freq = self.get_percentage(word, text_list) - if freq > thresh: - if random.randint(0, 1) == 0 and len(facts) <= 5: - facts.append( - fact_str.format(round(freq * 100))) - else: - rand = 0 - while rand == 0: - rand = random.randint(-(thresh*10), thresh*10) - lies.append( - fact_str.format(round(freq * 100 + rand))) - return facts, lies - - def check_uses(self, word, freq, thresh, fact_str, facts, lies, which): - quant = self.get_num_uses(word, freq) - if which == 0: - if quant >= thresh and len(facts) <= self.truths: - facts.append( - fact_str.format(quant)) - else: - if quant >= thresh and len(lies) <= 15 - self.truths: - lies.append( - fact_str.format(quant * random.randint(round(thresh/3), - thresh*3))) - return facts, lies + def percentage_check(self, freqmap, N): + str_build = "You use the word '{0}' in approximately {1}% of your facebook posts" + new_facts = [] + new_lies = [] + words = self.partywords + self.emowords + for word in words: + if word in freqmap: + freq = freqmap[word] / float(N) + if freq < .05: + continue + if random.randint(0, 1) == 0: + new_facts.append( + str_build.format(word, round(freq * 100))) + else: + rand = 0 + while rand == 0: + rand = random.randint(-int(freq*80), int(freq*200)) + new_lies.append( + str_build.format(word, round(freq * 100 + rand))) + return new_facts, new_lies + + def check_uses(self, freqmap): + str_build = "You used the word '{0}' at least {1} times on Facebook" + new_facts = [] + new_lies = [] + words = self.partywords + self.emowords + self.healthwords + self.poswork + for word in words: + if word in freqmap: + freq = freqmap[word] + if freq < 5: + continue + if random.randint(0, 1) == 0: + new_facts.append( + str_build.format(word, freq)) + else: + rand = 0 + while rand == 0: + rand = random.randint(-round(freq*.8), freq*3) + new_lies.append( + str_build.format(word, freq + rand)) + return new_facts, new_lies @gen.coroutine def process(self, user_data): @@ -166,55 +161,50 @@ def process(self, user_data): if user_data.data['gmail'].get('text'): gwords = self.get_words(user_data.data['gmail']['text']) gfreq = self.word_freq(gwords) - if len(gfreq) > 0: + if gfreq: if random.randint(0, 1) == 0: + most_common = gfreq.most_common(1) truth_data['true'].append( "Besides articles, prepositions, and pronouns your most " - "common word in email is {0}".format(gfreq[0][1])) + "common word in email is {0}".format(most_common[0][0])) else: - word_len = len(gfreq) - grab = round(word_len * .5) + fake = random.choice(list(gfreq.keys())) truth_data['false'].append( "Besides articles, prepositions, and pronouns your most " - "common word in email is \"{0}\"".format(gfreq[grab][1])) + "common word in email is \"{0}\"".format(fake)) - if user_data.data.get('fbtext'): - text_list = [post['text'] - for post in user_data.data['fbtext']['text']] + if user_data.data.get('fbtext') and user_data.data['fbtext']['text']: + text_list = (post['text'] + for post in user_data.data['fbtext']['text']) + N = len(user_data.data['fbtext']['text']) fbwords = self.get_words(text_list) fbfreq = self.word_freq(fbwords) # Later this can be a loop that tried different word, token pairs - truth_data['true'], truth_data['false'] = self.percentage_check( - 'me', text_list, .1, - "You use the word \"me\" in {0}% of your facebook posts", - truth_data['true'], truth_data['false']) - truth_data['true'], truth_data['false'] = self.percentage_check( - 'fuck', text_list, .05, - "You use the word \"fuck\" in {0}% of your facebook posts", - truth_data['true'], truth_data['false']) - # Now going to get into checking number of word uses - truth_data['true'], truth_data['false'] = self.check_uses( - 'yass', fbfreq, 5, - "You used the word \"yass\" {0} times on facebook", - truth_data['true'], truth_data['false'], 0) - truth_data['true'], truth_data['false'] = self.check_uses( - 'dafuq', fbfreq, 5, - "You used the word \"dafuq\" {0} times on facebook", - truth_data['true'], truth_data['false'], 0) - truth_data['true'], truth_data['false'] = self.check_uses( - 'lol', fbfreq, 5, - "You used the word \"LOL\" {0} times on facebook", - truth_data['true'], truth_data['false'], 1) - truth_data['true'], truth_data['false'] = self.check_uses( - 'love', fbfreq, 5, - "You used the word \"love\" {0} times on facebook", - truth_data['true'], truth_data['false'], 0) + if fbfreq: + if random.randint(0, 1) == 0: + most_common = fbfreq.most_common(1) + truth_data['true'].append( + "Besides articles, prepositions, and pronouns your most " + "common word on Facebook is {0}".format(most_common[0][0])) + else: + fake = random.choice(list(fbfreq.keys())) + truth_data['true'].append( + "Besides articles, prepositions, and pronouns your most " + "common word on Facebook is {0}".format(fake)) + + new_facts, new_lies = self.percentage_check(fbfreq, N) + truth_data['true'] += new_facts + truth_data['false'] += new_lies + new_facts, new_lies = self.check_uses(fbfreq) + truth_data['true'] += new_facts + truth_data['false'] += new_lies + if user_data.data.get('reddit'): try: fact, lie = self.common_subreddit( user_data.data['reddit']['submissions']) - if (len(truth_data['true']) < len(truth_data['false'])): + if (len(truth_data['true']) <= len(truth_data['false'])): truth_data['true'].append( "Your most common subreddit you submit to is {0}".format( fact)) @@ -225,8 +215,17 @@ def process(self, user_data): except IndexError: pass - truth_data['true'] = self.fill_truths(truth_data['true']) - truth_data['false'] = self.fill_lies(truth_data['false']) + random.shuffle(truth_data['true']) + random.shuffle(truth_data['false']) + self.logger.info("True: %d, False: %d", len(truth_data['true']), len(truth_data['false'])) + if len(truth_data['true']) < 8: + missing = 8 - len(truth_data['true']) + truth_data['true'] += random.sample( + self.realfacts, missing) + if len(truth_data['false']) < 8: + missing = 8 - len(truth_data['false']) + truth_data['false'] += random.sample( + self.fakefacts, missing) self.save_user_blob(truth_data, user_data) self.logger.info("Saved truth game data") @@ -241,6 +240,17 @@ def truth_grab(self, user, request): Returns relevant data that the exhibits may want to know """ data = self.load_user_blob(user) + rando = random.randint(0, 1) + if rando == 1: + if len(data['true']) > 7: + data['true'] = random.sample(data['true'], 7) + if len(data['false']) > 8: + data['false'] = random.sample(data['false'], 8) + else: + if len(data['true']) > 8: + data['true'] = random.sample(data['true'], 8) + if len(data['false']) > 7: + data['false'] = random.sample(data['false'], 7) return data @process_api_handler diff --git a/lib/rfidb.py b/lib/rfidb.py index 7fad6f6..9a72e23 100644 --- a/lib/rfidb.py +++ b/lib/rfidb.py @@ -1,6 +1,7 @@ from tornado import gen import rethinkdb as r +from collections import Counter from .dbhelper import RethinkDB from .user import User @@ -53,6 +54,8 @@ def delete_user(self, user_data): def associate_user(self, userid, rfid): conn = yield self.connection() try: + yield r.table('rfid').filter({"rfid": rfid}) \ + .update({'rfid': None}).run(conn) result = yield r.table('rfid').get(userid).update({'rfid': rfid}) \ .run(conn) if result['skipped'] == 1: diff --git a/lib/scrapers/__init__.py b/lib/scrapers/__init__.py index 754ec0e..94e6696 100644 --- a/lib/scrapers/__init__.py +++ b/lib/scrapers/__init__.py @@ -30,18 +30,22 @@ ] +@gen.coroutine +def _scrape(s, user_data): + try: + s.logger.info("Starting to scrape: " + user_data.userid) + result = yield s.scrape(user_data) + if not result: + s.logger.info("Scraped no data: " + user_data.userid) + else: + s.logger.info("Scraped data sucessfully: " + user_data.userid) + except Exception: + result = None + s.logger.exception("Scraper failed on user: " + user_data.userid) + return result + + @gen.coroutine def scrape(user_data): - data = {} - for s in scrapers: - try: - s.logger.info("Starting to scrape: " + user_data.userid) - data[s.name] = yield s.scrape(user_data) - if not data[s.name]: - s.logger.info("Scraped no data: " + user_data.userid) - else: - s.logger.info("Scraped data sucessfully: " + user_data.userid) - except Exception: - data[s.name] = None - s.logger.exception("Scraper failed on user: " + user_data.userid) + data = yield {s.name: _scrape(s, user_data) for s in scrapers} return data diff --git a/lib/scrapers/fbphotos.py b/lib/scrapers/fbphotos.py index 868158f..4068047 100644 --- a/lib/scrapers/fbphotos.py +++ b/lib/scrapers/fbphotos.py @@ -15,8 +15,8 @@ class FBPhotosScraper(BaseScraper): @property def num_images_per_user(self): if CONFIG['_mode'] == 'prod': - return 1000 - return 250 + return 200 + return 50 @gen.coroutine def scrape(self, user_data): @@ -75,18 +75,19 @@ def get_friends_profile(self, graph): 'feed', fields='from' ), - max_results=None, + max_results=self.num_images_per_user, ) friends = {d['from']['id']: d['from']['name'] for d in friends_raw} pictures = [] profile_pic = 'http://graph.facebook.com/{}/picture?type=large' - for fid, fname in friends.items(): + for i, (fid, fname) in enumerate(friends.items()): photo = { 'url': profile_pic.format(fid), 'id': 'img-' + fid, } + self.logger.debug("Friend face finding: %d / %d", i, len(friends)) faces = yield find_faces_url(photo['url'], hash_face=True) # go through the faces _we_ found and interpolate those results # with the tags from the image @@ -98,7 +99,7 @@ def get_friends_profile(self, graph): @gen.coroutine def parse_photos(self, graph, photos): - for photo in photos: + for i, photo in enumerate(photos): if 'tags' not in photo: continue photo['images'].sort(key=lambda x: x['height']*x['width']) @@ -113,6 +114,7 @@ def parse_photos(self, graph, photos): for t in photo['tags']['data'] if t.get('x') and t.get('y') ] + self.logger.debug("Face finding: %d / %d", i, len(photos)) faces = yield find_faces_url(image_url['source'], hash_face=True) # go through the faces _we_ found and interpolate those results # with the tags from the image diff --git a/lib/scrapers/gmailscrape.py b/lib/scrapers/gmailscrape.py index 7a9d7ba..5616d07 100644 --- a/lib/scrapers/gmailscrape.py +++ b/lib/scrapers/gmailscrape.py @@ -65,6 +65,7 @@ def get_content(self, raw): email_dict['body'] = body return email_dict + @gen.coroutine def paginate_messages(self, service, response, max_results=None): threads = [] if 'messages' in response: @@ -80,6 +81,7 @@ def paginate_messages(self, service, response, max_results=None): threads.append(i['threadId']) if max_results and len(threads) >= max_results: break + yield gen.sleep(0) return threads def get_recipient(self, email_data): @@ -167,7 +169,7 @@ def scrape(self, user_data): res = gmail.users().messages().list( userId='me', q='in:sent {0}'.format(token)).execute() - thread_ids_per_token[token] = self.paginate_messages( + thread_ids_per_token[token] = yield self.paginate_messages( gmail, res, max_results=self.num_threads @@ -188,4 +190,5 @@ def scrape(self, user_data): data['text'].append(body) data['snippets'].append(snippet) data['people'].append(people) + yield gen.sleep(0) return data diff --git a/lib/scrapers/lib/utils.py b/lib/scrapers/lib/utils.py index 897fa13..5a8008d 100644 --- a/lib/scrapers/lib/utils.py +++ b/lib/scrapers/lib/utils.py @@ -43,6 +43,7 @@ def apiclient_paginate(resource, action, params, http=None, max_results=500): def facebook_paginate(data, max_results=500): paginated_data = [] while True: + logger.debug("Paginating facebook data: %d/%d", len(paginated_data), max_results) paginated_data.extend(data['data']) if max_results is not None and len(paginated_data) >= max_results: break diff --git a/lib/scrapers/twitter.py b/lib/scrapers/twitter.py index 459276f..728dbaf 100644 --- a/lib/scrapers/twitter.py +++ b/lib/scrapers/twitter.py @@ -43,6 +43,7 @@ def all_following(self, api): 'description': friend_meta.description, } data.append(user) + yield gen.sleep(0) return data @gen.coroutine @@ -57,6 +58,7 @@ def all_tweets(self, api, max_tweets): break data.extend(t.text for t in tweets) max_id = tweets.max_id + yield gen.sleep(0) return data @gen.coroutine diff --git a/lib/userprocess.py b/lib/userprocess.py index 7429541..d826d62 100644 --- a/lib/userprocess.py +++ b/lib/userprocess.py @@ -1,21 +1,17 @@ from tornado import gen -import pickle from . import scrapers from . import processors -from .config import CONFIG @gen.coroutine -def userprocess(user): - if CONFIG.get('_mode') == 'dev': - try: - fname = './data/debug/{}.pkl'.format(user.userid) - with open(fname, 'rb') as fd: - user.data = pickle.load(fd) - print("Found debug data for user: ", user.userid) - except IOError: - pass - if not user.data: - user.data = yield scrapers.scrape(user) - user.process_results = yield processors.process(user) +def userprocess(users, use_cache=True): + for user in users: + if use_cache: + try: + user.data = processors.DebugProcessor().load_scrape(user) + except IOError: + pass + if not user.data: + user.data = yield scrapers.scrape(user) + user.process_results = yield processors.process(user) diff --git a/scripts/actors_rfid.py b/scripts/actors_rfid.py new file mode 100644 index 0000000..bfceaee --- /dev/null +++ b/scripts/actors_rfid.py @@ -0,0 +1,54 @@ +import rethinkdb as r +actors = { + "Felicity Turner": "6b8b8d4f-fe8b-4b0e-b257-973b17e343c5", + "Desiree Harlech": "b1f52988-0971-42ab-a515-7c9b0d9f8d9c", + "Amelia Bloom": "1002c758-8005-4b99-ba96-38c0faacffeb", + "Don DeClaire": "67d37fb5-261b-41da-8d3f-2e5aff93ead1", + "Bo Rakenfold": "1423bf50-6bfb-4eb6-997e-25dd1e801a98", + "Lily Jordan": "244f62b1-ee51-444d-84c2-b72a39a2fdbd", + "hacker": "hacker", + "email": "email", +} + + +def create_rfid(name, user_id): + return { + "id": user_id, + "publickey": None, + "privatekey": None, + "showid": None, + "showdate": None, + "name": name, + "rfid": None, + } + + +def create_perm(name, user_id): + return { + 'name': name, + 'id': user_id, + 'delete_processor': False, + 'amelia_processor': True, + 'debug_processor': True, + 'interview_processor': True, + 'mental_health': True, + 'mirror_processor': True, + 'news_processor': True, + 'ownup': True, + 'pr0n_processor': True, + 'recommend_processor': True, + 'romance_processor': True, + 'tos_processor': True, + 'tracked_processor': True, + 'truth_processor': True + } + + +if __name__ == "__main__": + conn = r.connect(db='gulperbase') + for name, user_id in actors.items(): + print("Adding: ", name) + result = r.table("rfid").insert(create_rfid(name, user_id), conflict='update').run(conn) + print(result) + result = r.table("exhibitpermissions").insert(create_perm(name, user_id), conflict='update').run(conn) + print(result)