diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..db29874 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +*.py[c] + +build/* +dist/* +media/* +*.egg-info + +*.db + +*~ +*.swp + +.DS_Store + +.idea + +pip-log.txt + +favicon.ico +fql_data.py +fql_data.sql +graph_data.py +graph_data.sql +sqlparse +sqlparse-0.1.3.tar.gz +sqlparse-0.1.3/* diff --git a/app.py b/app.py new file mode 100644 index 0000000..b394e3b --- /dev/null +++ b/app.py @@ -0,0 +1,31 @@ +import webapp2 +import json + + +class TestUsersHandler(webapp2.RequestHandler): + ROUTES = [ + (r'/(\d+)/accounts/test-users', 'app.TestUsersHandler'), + ] + + @classmethod + def init(cls, conn, me): + """ + conn: sqlite3.Connection + """ + cls.conn = conn + + def get(self, app_id): + sql = """ + SELECT DISTINCT user_id, token + FROM oauth_access_tokens; + """ + cursor = self.conn.execute(sql) + rows = cursor.fetchall() + + results = {'data': []} + for row in rows: + user_id, token = row + results['data'].append({'id': user_id, 'access_token': token}) + + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' + json.dump(results, self.response.out, indent=2) diff --git a/download.py b/download.py index 594a645..c91a560 100755 --- a/download.py +++ b/download.py @@ -42,21 +42,16 @@ __author__ = ['Ryan Barrett '] +import requests import collections -import httplib -import itertools import json import logging -import operator import optparse import re import sys -import traceback import urllib -import urllib2 import urlparse -import graph import schemautil @@ -107,6 +102,7 @@ 'number': ('int', 'INTEGER'), 'object': ('object', ''), 'string': ('string', 'TEXT'), + 'photo': ('string', 'TEXT'), 'structure': ('object', ''), 'time': ('int', 'INTEGER'), 'uid': ('int', 'INTEGER'), @@ -293,35 +289,7 @@ def print_and_flush(str): sys.stdout.flush() -def urlopen_with_retries(url, data=None): - """Wrapper for urlopen that automatically retries on HTTP errors. - - If redirect is False and the url is 302 redirected, raises Redirected - with the destination URL in the exception value. - """ - for i in range(HTTP_RETRIES + 1): - try: - opened = urllib2.urlopen(url, data=data, timeout=HTTP_TIMEOUT_S) - # if we ever need to determine whether we're redirected here, do something - # like this: - # - # if opened.geturl() != url: - # ... - # - # it's not great - you can easily imagine failure cases - but it's by far - # the simplest way. discussion: http://stackoverflow.com/questions/110498 - return opened - - except (IOError, urllib2.HTTPError), e: - logging.debug('retrying due to %s' % e) - - print >> sys.stderr, 'Gave up on %s after %d tries. Last error:' % ( - url, HTTP_RETRIES) - traceback.print_exc(file=sys.stderr) - raise e - - -def make_column(table, column, fb_type, indexable=None): +def make_column(table, column, raw_fb_type, indexable=None): """Populates and returns a Column for a schema. Args: @@ -332,10 +300,10 @@ def make_column(table, column, fb_type, indexable=None): Returns: Column """ - fb_type, sqlite_type = COLUMN_TYPES.get(fb_type.lower(), (None, None)) - if fb_type is None: - print >> sys.stderr, 'TODO: %s.%s has unknown type %s' % ( - table, column, raw_fb_type) + fb_type, sqlite_type = COLUMN_TYPES.get(raw_fb_type.lower(), (None, None)) + + if raw_fb_type is None: + print >> sys.stderr, 'TODO: %s.%s has unknown type %s' % (table, column, raw_fb_type) return schemautil.Column(name=column, fb_type=fb_type, @@ -354,6 +322,7 @@ def column_from_metadata_field(table, field): """ name = field['name'] match = GRAPH_DESCRIPTION_TYPE_RE.search(field['description']) + if match: fb_type = match.group(1) else: @@ -375,12 +344,12 @@ def scrape_schema(schema, url, column_re): """ print_and_flush('Generating %s' % schema.__class__.__name__) - index_html = urlopen_with_retries(url).read() + index_html = requests.get(url).content print_and_flush('.') links_html = TABLE_LINKS_RE.search(index_html).group() for link in TABLE_LINK_RE.findall(links_html): - table_html = urlopen_with_retries(link).read() + table_html = requests.get(link).content tables = TABLE_RE.findall(table_html) assert len(tables) == 1 table = tables[0].strip() @@ -391,7 +360,7 @@ def scrape_schema(schema, url, column_re): # column_re has three groups: indexable, name, type column_data = column_re.findall(table_html) column_names = [c[1] for c in column_data] - override_types = OVERRIDE_COLUMN_TYPES[table] + override_indexable = OVERRIDE_COLUMN_INDEXABLE[table] for name in set(override_types.keys()) | set(override_indexable.keys()): if name not in column_names: @@ -494,13 +463,17 @@ def fetch_graph_schema_and_data(ids): print_and_flush('Generating Graph API schema and example data') # fetch the objects - objects = batch_request(ids, args={'metadata': 'true', - 'limit': options.num_per_type}) + objects = batch_request(ids, args={'metadata': 'true', 'limit': options.num_per_type}) # strip the metadata and generate and store the schema connections = [] # list of (name, url) tuples for id, object in objects.items(): - metadata = object.pop('metadata') + try: + metadata = object.pop('metadata') + except AttributeError: + if isinstance(object, bool): + continue + raise table = object['type'] # columns @@ -517,7 +490,7 @@ def fetch_graph_schema_and_data(ids): # store the objects in the dataset dataset.data = dict( (id, schemautil.Data(table=object['type'], query=id, data=object)) - for id, object in objects.items()) + for id, object in objects.items() if not isinstance(object, bool)) conn_paths = [urlparse.urlparse(url).path for name, url in connections if name not in UNSUPPORTED_CONNECTIONS] @@ -541,34 +514,6 @@ def fetch_graph_schema_and_data(ids): return schema, dataset -# this code works fine, but it's been replaced with batch_request(). -# it's still good though. keep it or dump it? -# -# def facebook_query(url=None, args=None, query=None, table=None): -# """Makes an FQL or Graph API request. - -# Args: -# url: string -# args: dict of query parameters -# query: value for the query field in the returned Data object -# table: string - -# Returns: -# schemautil.Data -# """ -# parts = list(urlparse.urlparse(url)) -# args['access_token'] = options.access_token -# for arg, vals in urlparse.parse_qs(parts[4]).items(): -# args[arg] = vals[0] -# parts[4] = urllib.urlencode(args) - -# url = urlparse.urlunparse(parts) -# result = json.loads(urlopen_with_retries(url).read()) -# assert 'error_code' not in result, 'FQL error:\n%s' % result -# url = re.sub('access_token=[^&]+', 'access_token=XXX', url) - -# return schemautil.Data(table=table, query=query, url=url, data=result) - def batch_request(urls, args=None): """Makes a Graph API batch request. @@ -586,17 +531,20 @@ def batch_request(urls, args=None): urls = list(urls) params = '?%s' % urllib.urlencode(args) if args else '' - requests = [{'method': 'GET', 'relative_url': url + params} for url in urls] - + requests_to_do = [{'method': 'GET', 'relative_url': url + params} for url in urls] responses = [] - for i in range(0, len(requests), MAX_REQUESTS_PER_BATCH): - data = urllib.urlencode({'access_token': options.access_token, - 'batch': json.dumps(requests[i:i + 50])}) - response = urlopen_with_retries(options.graph_api_url, data=data) - responses.extend(json.loads(response.read())) + + for i in range(0, len(requests_to_do), MAX_REQUESTS_PER_BATCH): + data = { + 'access_token': options.access_token, + 'batch': json.dumps(requests_to_do[i:i + 50]) + } + + response = requests.post(options.graph_api_url, data=data) + responses.extend(json.loads(response.content)) print_and_flush('.') - assert len(responses) == len(requests) + assert len(responses) == len(requests_to_do) results = {} for url, resp in zip(urls, responses): @@ -681,8 +629,13 @@ def main(): options = parse_args() if options.db_file: # FIXME - should do dupe checking - sql = 'INSERT INTO oauth_access_tokens(code, token) VALUES("asdf", "%s");' % options.access_token - schemautil.get_db(options.db_file).executescript(sql) + response = requests.get('https://graph.facebook.com/me', params={'access_token': options.access_token}) + if response.ok: + user_id = response.json['id'] + sql = 'INSERT INTO oauth_access_tokens(user_id, code, token) VALUES("%s", "asdf", "%s");' % (user_id, options.access_token) + schemautil.get_db(options.db_file).executescript(sql) + else: + print >> sys.stderr, "There was a problem downloading the user info [%s]" % options.access_token if options.fql_schema: fql_schema = schemautil.FqlSchema() diff --git a/fql.py b/fql.py index 2c6ed8f..6f73027 100644 --- a/fql.py +++ b/fql.py @@ -264,14 +264,28 @@ def get(self): try: query_arg = 'q' if graph_endpoint else 'query' + + # Multiline, multispace support query = self.request.get(query_arg) + query = re.sub(' +', ' ', query).replace('\n', '').strip() + if not query: raise MissingParamError(query_arg) - token = self.request.get('access_token') + token = self.request.get('access_token') if token and not oauth.AccessTokenHandler.is_valid_token(self.conn, token): raise InvalidAccessTokenError() + # Find current me if not provided + if not self.me and token: + try: + sql = 'SELECT user_id FROM oauth_access_tokens WHERE token="%s";' % token + + cursor = self.conn.execute(sql) + self.me = cursor.fetchone()[0] + except: + pass + logging.debug('Received FQL query: %s' % query) fql = Fql(self.schema, query, self.me) @@ -291,6 +305,10 @@ def get(self): except FqlError, e: results = self.error(self.request.GET, e.code, e.msg) + # Encapsulate results in a data keyword + if graph_endpoint: + results = {'data': results} + if self.request.get('format') == 'json' or graph_endpoint: json.dump(results, self.response.out, indent=2) else: diff --git a/graph.py b/graph.py index a55e873..18c91b9 100644 --- a/graph.py +++ b/graph.py @@ -15,13 +15,12 @@ import types import urllib import re - +from urllib import unquote import datetime import random import sys import webapp2 - import oauth import schemautil @@ -336,10 +335,21 @@ def _get(self, id, connection): id = None try: - token = self.request.get('access_token') + token = self.request.get('access_token') + if token and not oauth.AccessTokenHandler.is_valid_token(self.conn, token): raise ValidationError() + # Find current me if not provided + if not self.me and token: + try: + sql = 'SELECT user_id FROM oauth_access_tokens WHERE token="%s";' % token + + cursor = self.conn.execute(sql) + self.me = cursor.fetchone()[0] + except: + pass + namedict = self.prepare_ids(id) if connection: @@ -460,16 +470,22 @@ def get_objects(self, namedict): cursor = self.conn.execute( - 'SELECT id, data FROM graph_objects WHERE id IN (%s)' % self.qmarks(ids), + 'SELECT DISTINCT id, data FROM graph_objects WHERE id IN (%s)' % self.qmarks(ids), ids) - ret_dict = dict((namedict[obj_id], json.loads(data)) for obj_id, data in cursor.fetchall()) + + # List of fields to filter + fields = unquote(self.request.get('fields')) + fields = fields.split(',') if fields else [] + filtered_data = {} + for user_id, data in cursor.fetchall(): + filtered_data[user_id] = dict([[key, value] for key, value in json.loads(data).items() if key in fields or not fields]) # Anything in the published graph objects overwrite the normal results for obj_id in ids: if obj_id in GraphHandler.posted_graph_objects: - ret_dict[obj_id] = GraphHandler.posted_graph_objects[obj_id] + filtered_data[obj_id] = GraphHandler.posted_graph_objects[obj_id] - return ret_dict + return filtered_data def get_connections(self, namedict, connection): if not namedict: @@ -478,7 +494,7 @@ def get_connections(self, namedict, connection): raise UnknownPathError(connection) ids = namedict.keys() - query = ('SELECT id, data FROM graph_connections ' + query = ('SELECT DISTINCT id, data FROM graph_connections ' 'WHERE id IN (%s) AND connection = ?' % self.qmarks(ids)) cursor = self.conn.execute(query, ids + [connection]) rows = cursor.fetchall() @@ -492,9 +508,18 @@ def get_connections(self, namedict, connection): posted_data = GraphHandler.posted_connections.get(name, {}).get(connection, []) resp[name] = {"data": posted_data} - for id, data in rows: - resp[namedict[id]]['data'].extend(posted_data) - resp[namedict[id]]['data'].append(json.loads(data)) + # Anything in the published graph objects overwrite the normal results + for obj_id in ids: + if obj_id in GraphHandler.posted_graph_objects: + filtered_data[obj_id] = GraphHandler.posted_graph_objects[obj_id] + + + # List of fields to filter + fields = unquote(self.request.get('fields')) + fields = fields.split(',') if fields else [] + for user_id, data in rows: + filtered_data = dict([[key, value] for key, value in json.loads(data).items() if key in fields or not fields]) + resp[namedict[user_id]]['data'].append(filtered_data) return resp @@ -531,7 +556,7 @@ def prepare_ids(self, path_id): qmarks = self.qmarks(names) cursor = self.conn.execute( - 'SELECT id, alias FROM graph_objects WHERE id IN (%s) OR alias IN (%s)' % + 'SELECT DISTINCT id, alias FROM graph_objects WHERE id IN (%s) OR alias IN (%s)' % (qmarks, qmarks), tuple(names) * 2) diff --git a/graph_schema.py b/graph_schema.py index fac65b4..49efced 100644 --- a/graph_schema.py +++ b/graph_schema.py @@ -1,6 +1,6 @@ # Do not edit! Generated automatically by mockfacebook. # https://github.com/rogerhu/mockfacebook -# 2011-11-08 16:25:41.357079 +# 2012-05-15 12:22:56.585008 {'connections': {u'album': [u'photos', u'likes', u'comments'], u'application': [u'feed', @@ -16,8 +16,6 @@ u'events', u'statuses', u'insights'], - u'checkin': [u'likes', u'comments'], - u'comment': [u'likes', u'comments'], u'event': [u'feed', u'picture', u'noreply', @@ -25,23 +23,23 @@ u'invited', u'attending', u'declined'], - u'friendlist': [u'members'], u'group': [u'feed', u'picture', u'members', u'docs'], - u'link': [u'likes', u'comments'], u'note': [u'comments'], u'page': [u'feed', u'tagged', + u'milestones', u'videos', u'links', u'notes', u'posts', u'photos', + u'offers', u'questions', u'albums', u'events', u'statuses'], - u'photo': [u'likes', u'comments'], - u'status': [u'likes', u'comments'], + u'photo': [u'likes', u'comments', u'tags'], + u'status': [u'likes', u'comments', u'tags'], u'user': [u'feed', u'tagged', u'family', @@ -91,11 +89,11 @@ Column(name=u'count', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'type', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'created_time', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'updated_time', fb_type='string', sqlite_type='TEXT', indexable=None)], + Column(name=u'updated_time', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'can_upload', fb_type='bool', sqlite_type='INTEGER', indexable=None)], u'application': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'name', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'description', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'canvas_name', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'category', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'company', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'icon_url', fb_type='string', sqlite_type='TEXT', indexable=None), @@ -107,24 +105,39 @@ Column(name=u'monthly_active_users', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'migrations', fb_type='array', sqlite_type='', indexable=None), Column(name=u'namespace', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'restrictions', fb_type='object', sqlite_type='', indexable=None)], - u'checkin': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'from', fb_type='object', sqlite_type='', indexable=None), - Column(name=u'tags', fb_type='array', sqlite_type='', indexable=None), - Column(name=u'place', fb_type='object', sqlite_type='', indexable=None), - Column(name=u'application', fb_type='object', sqlite_type='', indexable=None), - Column(name=u'created_time', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'likes', fb_type='array', sqlite_type='', indexable=None), - Column(name=u'message', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'comments', fb_type='array', sqlite_type='', indexable=None), - Column(name=u'type', fb_type='string', sqlite_type='TEXT', indexable=None)], - u'comment': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'from', fb_type='object', sqlite_type='', indexable=None), - Column(name=u'message', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'created_time', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'likes', fb_type='int', sqlite_type='INTEGER', indexable=None), - Column(name=u'user_likes', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'type', fb_type='string', sqlite_type='TEXT', indexable=None)], + Column(name=u'restrictions', fb_type='object', sqlite_type='', indexable=None), + Column(name=u'app_domains', fb_type='array', sqlite_type='', indexable=None), + Column(name=u'auth_dialog_data_help_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'auth_dialog_description', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'auth_dialog_headline', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'auth_dialog_perms_explanation', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'auth_referral_user_perms', fb_type='array', sqlite_type='', indexable=None), + Column(name=u'auth_referral_friend_perms', fb_type='array', sqlite_type='', indexable=None), + Column(name=u'auth_referral_default_activity_privacy', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'auth_referral_enabled', fb_type='bool', sqlite_type='INTEGER', indexable=None), + Column(name=u'auth_referral_extended_perms', fb_type='array', sqlite_type='', indexable=None), + Column(name=u'auth_referral_response_type', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'canvas_fluid_height', fb_type='bool', sqlite_type='INTEGER', indexable=None), + Column(name=u'canvas_fluid_width', fb_type='bool', sqlite_type='INTEGER', indexable=None), + Column(name=u'canvas_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'contact_email', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'created_time', fb_type='int', sqlite_type='INTEGER', indexable=None), + Column(name=u'creator_uid', fb_type='int', sqlite_type='INTEGER', indexable=None), + Column(name=u'deauth_callback_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'iphone_app_store_id', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'hosting_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'mobile_web_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'page_tab_default_name', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'page_tab_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'privacy_policy_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'secure_canvas_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'secure_page_tab_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'server_ip_whitelist', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'social_discovery', fb_type='bool', sqlite_type='INTEGER', indexable=None), + Column(name=u'terms_of_service_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'user_support_email', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'user_support_url', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'website_url', fb_type='string', sqlite_type='TEXT', indexable=None)], u'domain': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'name', fb_type='string', sqlite_type='TEXT', indexable=None)], u'event': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), @@ -137,9 +150,6 @@ Column(name=u'venue', fb_type='object', sqlite_type='', indexable=None), Column(name=u'privacy', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'updated_time', fb_type='string', sqlite_type='TEXT', indexable=None)], - u'friendlist': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'name', fb_type='string', sqlite_type='TEXT', indexable=None), - Column(name=u'type', fb_type='string', sqlite_type='TEXT', indexable=None)], u'group': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'version', fb_type='int', sqlite_type='INTEGER', indexable=None), Column(name=u'icon', fb_type='string', sqlite_type='TEXT', indexable=None), @@ -161,10 +171,16 @@ Column(name=u'name', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'link', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'category', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'is_published', fb_type='bool', sqlite_type='INTEGER', indexable=None), + Column(name=u'can_post', fb_type='bool', sqlite_type='INTEGER', indexable=None), Column(name=u'likes', fb_type='int', sqlite_type='INTEGER', indexable=None), Column(name=u'location', fb_type='object', sqlite_type='', indexable=None), Column(name=u'phone', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'checkins', fb_type='int', sqlite_type='INTEGER', indexable=None), + Column(name=u'picture', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'cover', fb_type=None, sqlite_type=None, indexable=None), + Column(name=u'website', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'talking_about_count', fb_type='int', sqlite_type='INTEGER', indexable=None), Column(name=u'access_token', fb_type='string', sqlite_type='TEXT', indexable=None)], u'photo': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'from', fb_type='object', sqlite_type='', indexable=None), @@ -177,12 +193,14 @@ Column(name=u'width', fb_type='int', sqlite_type='INTEGER', indexable=None), Column(name=u'images', fb_type='array', sqlite_type='', indexable=None), Column(name=u'link', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'place', fb_type='object', sqlite_type='', indexable=None), Column(name=u'created_time', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'updated_time', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'position', fb_type='int', sqlite_type='INTEGER', indexable=None)], u'status': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'from', fb_type='object', sqlite_type='', indexable=None), Column(name=u'message', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'place', fb_type='object', sqlite_type='', indexable=None), Column(name=u'updated_time', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'type', fb_type='string', sqlite_type='TEXT', indexable=None)], u'user': [Column(name=u'id', fb_type='string', sqlite_type='TEXT', indexable=None), @@ -196,11 +214,13 @@ Column(name=u'link', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'username', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'third_party_id', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'installed', fb_type='object', sqlite_type='', indexable=None), Column(name=u'timezone', fb_type='int', sqlite_type='INTEGER', indexable=None), Column(name=u'updated_time', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'verified', fb_type='bool', sqlite_type='INTEGER', indexable=None), Column(name=u'bio', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'birthday', fb_type='string', sqlite_type='TEXT', indexable=None), + Column(name=u'cover', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'education', fb_type='array', sqlite_type='', indexable=None), Column(name=u'email', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'hometown', fb_type='object', sqlite_type='', indexable=None), @@ -209,6 +229,7 @@ Column(name=u'political', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'favorite_athletes', fb_type='array', sqlite_type='', indexable=None), Column(name=u'favorite_teams', fb_type='array', sqlite_type='', indexable=None), + Column(name=u'picture', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'quotes', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'relationship_status', fb_type='string', sqlite_type='TEXT', indexable=None), Column(name=u'religion', fb_type='string', sqlite_type='TEXT', indexable=None), diff --git a/mockfacebook.sql b/mockfacebook.sql index 24518ec..fad14c4 100644 --- a/mockfacebook.sql +++ b/mockfacebook.sql @@ -8,6 +8,7 @@ CREATE TABLE IF NOT EXISTS oauth_codes ( ); CREATE TABLE IF NOT EXISTS oauth_access_tokens ( + user_id TEXT NOT NULL, token TEXT NOT NULL, code TEXT NOT NULL, FOREIGN KEY(code) REFERENCES auth_codes(code) diff --git a/schemautil.py b/schemautil.py index 91d46ca..2b2c3d5 100644 --- a/schemautil.py +++ b/schemautil.py @@ -339,7 +339,10 @@ def to_sql(self): for col in self.schema.tables[table]) for object in data.data: # order columns to match schema (which is the order in FQL docs) - values_str = self.schema.json_to_sqlite(object, table) + try: + values_str = self.schema.json_to_sqlite(object, table) + except: + continue output.append("""\ INSERT OR IGNORE INTO `%s` ( %s diff --git a/server.py b/server.py index 16b0af9..a183b8d 100755 --- a/server.py +++ b/server.py @@ -19,6 +19,7 @@ import webapp2 import fql +import app import graph import oauth import schemautil @@ -35,6 +36,7 @@ # order matters here! the first handler with a matching route is used. HANDLER_CLASSES = ( + app.TestUsersHandler, oauth.AuthCodeHandler, oauth.AccessTokenHandler, fql.FqlHandler, @@ -59,7 +61,7 @@ def parse_args(argv): help='port to serve on (default %default)') parser.add_option('-f', '--db_file', default=schemautil.DEFAULT_DB_FILE, help='SQLite database file (default %default)') - parser.add_option('--me', type='str', default=1, + parser.add_option('--me', type='str', default='', help='user id that me() should return (default %default)') options, args = parser.parse_args(args=argv)