diff --git a/.gitignore b/.gitignore index 65f61e9d4..e0d257fe9 100644 --- a/.gitignore +++ b/.gitignore @@ -16,9 +16,11 @@ .noseids /env_api /venv +/.venv *.swp config/docker .coverage coverage.xml /config/env.local +/docker-compose.override.local.yml project.tar diff --git a/c2corg_api/caching.py b/c2corg_api/caching.py index 8005cdc04..562edbd4d 100644 --- a/c2corg_api/caching.py +++ b/c2corg_api/caching.py @@ -33,6 +33,7 @@ def create_region(name): cache_document_info = create_region('info') cache_sitemap = create_region('sitemap') cache_sitemap_xml = create_region('sitemap_xml') +cache_outing_map = create_region('outing_map') caches = [ cache_document_cooked, @@ -42,7 +43,8 @@ def create_region(name): cache_document_version, cache_document_info, cache_sitemap, - cache_sitemap_xml + cache_sitemap_xml, + cache_outing_map ] diff --git a/c2corg_api/models/outing_map_queries.py b/c2corg_api/models/outing_map_queries.py new file mode 100644 index 000000000..b048556a5 --- /dev/null +++ b/c2corg_api/models/outing_map_queries.py @@ -0,0 +1,81 @@ +from sqlalchemy.sql.functions import func + +from c2corg_api.models import DBSession +from c2corg_api.models.document import DocumentGeometry, DocumentLocale +from c2corg_api.models.outing import Outing + + +def _bbox_envelope(bbox_3857): + xmin, ymin, xmax, ymax = bbox_3857 + return func.ST_MakeEnvelope(xmin, ymin, xmax, ymax, 3857) + + +def _filter_by_activities(query, activities): + if activities: + query = query.filter(Outing.activities.overlap(activities)) + return query + + +def get_outings_heatmap(bbox_3857, activities, cell_size_m): + """Returns density buckets `(x, y, count)` for outings whose + representative point (`geometry.geom`) falls in the given bbox, with + coordinates snapped to a grid of `cell_size_m` meters (EPSG:3857). + + Bucketing uses the outing's single representative point rather than + its track (`geom_detail`), so that one outing always contributes to + exactly one bucket (a "density of outings" heatmap, not a "density of + kilometers of track" one). + """ + envelope = _bbox_envelope(bbox_3857) + grid_point = func.ST_SnapToGrid(DocumentGeometry.geom, cell_size_m) + + query = ( + DBSession.query( + func.ST_X(grid_point).label('x'), + func.ST_Y(grid_point).label('y'), + func.count(Outing.document_id).label('count')) + .join( + DocumentGeometry, + DocumentGeometry.document_id == Outing.document_id) + .filter(Outing.redirects_to.is_(None)) + .filter(DocumentGeometry.geom.isnot(None)) + .filter(DocumentGeometry.geom.ST_Intersects(envelope))) + query = _filter_by_activities(query, activities) + + return query.group_by(grid_point).all() + + +def get_outings_tracks(bbox_3857, activities, limit): + """Returns up to `limit + 1` `(document_id, geom_detail)` rows for + outings whose track (`geometry.geom_detail`) intersects the given + bbox. Callers should treat a result of length `limit + 1` as "too many + results for this bbox" rather than displaying a silently-truncated + set. + """ + envelope = _bbox_envelope(bbox_3857) + + query = ( + DBSession.query(Outing.document_id, DocumentGeometry.geom_detail) + .join( + DocumentGeometry, + DocumentGeometry.document_id == Outing.document_id) + .filter(Outing.redirects_to.is_(None)) + .filter(DocumentGeometry.geom_detail.isnot(None)) + .filter(DocumentGeometry.geom_detail.ST_Intersects(envelope))) + query = _filter_by_activities(query, activities) + + return query.limit(limit + 1).all() + + +def get_locales_for_documents(document_ids): + """Returns all `DocumentLocale` rows for the given document ids, so + that callers can pick the best-matching locale per document (see + `c2corg_api.views.get_best_locale`). + """ + if not document_ids: + return [] + + return ( + DBSession.query(DocumentLocale) + .filter(DocumentLocale.document_id.in_(document_ids)) + .all()) diff --git a/c2corg_api/tests/views/test_outing_map.py b/c2corg_api/tests/views/test_outing_map.py new file mode 100644 index 000000000..8cb4016b1 --- /dev/null +++ b/c2corg_api/tests/views/test_outing_map.py @@ -0,0 +1,153 @@ +from datetime import date +from unittest.mock import patch + +from c2corg_api.caching import cache_outing_map +from c2corg_api.models.document import DocumentGeometry +from c2corg_api.models.outing import Outing, OutingLocale +from c2corg_api.tests.views import BaseTestRest + + +class TestOutingsHeatmapRest(BaseTestRest): + + def setUp(self): # noqa + super(TestOutingsHeatmapRest, self).setUp() + self._prefix = '/outings/map/heatmap' + # avoid cache hits from other tests reusing the same bbox + cache_outing_map.invalidate() + + self.session.add(Outing( + activities=['hiking'], date_start=date(2016, 1, 1), + date_end=date(2016, 1, 1), + locales=[OutingLocale(lang='en', title='Hike 1')], + geometry=DocumentGeometry( + geom='SRID=3857;POINT(635956 5723604)'))) + self.session.add(Outing( + activities=['hiking'], date_start=date(2016, 1, 2), + date_end=date(2016, 1, 2), + locales=[OutingLocale(lang='en', title='Hike 2')], + geometry=DocumentGeometry( + geom='SRID=3857;POINT(635960 5723610)'))) + self.session.add(Outing( + activities=['skitouring'], date_start=date(2016, 1, 3), + date_end=date(2016, 1, 3), + locales=[OutingLocale(lang='en', title='Ski tour')], + geometry=DocumentGeometry( + geom='SRID=3857;POINT(635960 5723620)'))) + # far away, should never be returned by the bbox used in tests + self.session.add(Outing( + activities=['hiking'], date_start=date(2016, 1, 4), + date_end=date(2016, 1, 4), + locales=[OutingLocale(lang='en', title='Far away hike')], + geometry=DocumentGeometry( + geom='SRID=3857;POINT(0 0)'))) + self.session.flush() + + self.bbox = '635000,5723000,637000,5724000' + + def test_missing_bbox(self): + body = self.app.get( + self._prefix + '?zoom=8', status=400).json + self.assertErrorsContain(body, 'bbox') + + def test_missing_zoom(self): + body = self.app.get( + self._prefix + '?bbox=' + self.bbox, status=400).json + self.assertErrorsContain(body, 'zoom') + + def test_invalid_bbox(self): + body = self.app.get( + self._prefix + '?bbox=1,2,3&zoom=8', status=400).json + self.assertErrorsContain(body, 'bbox') + + def test_invalid_activity(self): + body = self.app.get( + self._prefix + '?bbox=' + self.bbox + + '&zoom=8&act=cooking', status=400).json + self.assertErrorsContain(body, 'act') + + def test_heatmap(self): + body = self.app.get( + self._prefix + '?bbox=' + self.bbox + '&zoom=8', status=200) \ + .json + self.assertIn('cell_size', body) + self.assertIn('buckets', body) + total = sum(bucket['count'] for bucket in body['buckets']) + # only the 3 outings within the bbox are counted, not the one + # at (0, 0) + self.assertEqual(total, 3) + + def test_heatmap_filtered_by_activity(self): + body = self.app.get( + self._prefix + '?bbox=' + self.bbox + + '&zoom=8&act=skitouring', status=200).json + total = sum(bucket['count'] for bucket in body['buckets']) + self.assertEqual(total, 1) + + +class TestOutingsTracksRest(BaseTestRest): + + def setUp(self): # noqa + super(TestOutingsTracksRest, self).setUp() + self._prefix = '/outings/map/tracks' + # avoid cache hits from other tests reusing the same bbox + cache_outing_map.invalidate() + + self.outing1 = Outing( + activities=['hiking'], date_start=date(2016, 1, 1), + date_end=date(2016, 1, 1), + locales=[OutingLocale(lang='en', title='Hike 1')], + geometry=DocumentGeometry( + geom='SRID=3857;POINT(635956 5723604)', + geom_detail='SRID=3857;LINESTRING(635956 5723604, ' + '635966 5723644)')) + self.session.add(self.outing1) + + self.outing2 = Outing( + activities=['skitouring'], date_start=date(2016, 1, 2), + date_end=date(2016, 1, 2), + locales=[OutingLocale(lang='en', title='Ski tour')], + geometry=DocumentGeometry( + geom='SRID=3857;POINT(635960 5723610)', + geom_detail='SRID=3857;LINESTRING(635960 5723610, ' + '635970 5723650)')) + self.session.add(self.outing2) + + # no geom_detail: must never be returned by the tracks endpoint + self.session.add(Outing( + activities=['hiking'], date_start=date(2016, 1, 3), + date_end=date(2016, 1, 3), + locales=[OutingLocale(lang='en', title='No track')], + geometry=DocumentGeometry( + geom='SRID=3857;POINT(635958 5723606)'))) + self.session.flush() + + self.bbox = '635000,5723000,637000,5724000' + + def test_missing_bbox(self): + body = self.app.get(self._prefix, status=400).json + self.assertErrorsContain(body, 'bbox') + + def test_tracks(self): + body = self.app.get( + self._prefix + '?bbox=' + self.bbox, status=200).json + self.assertFalse(body['truncated']) + self.assertEqual(len(body['outings']), 2) + titles = {o['title'] for o in body['outings']} + self.assertEqual(titles, {'Hike 1', 'Ski tour'}) + for outing in body['outings']: + self.assertEqual(outing['type'], 'o') + self.assertIn('geom_detail', outing['geometry']) + + def test_tracks_filtered_by_activity(self): + body = self.app.get( + self._prefix + '?bbox=' + self.bbox + '&act=skitouring', + status=200).json + self.assertEqual(len(body['outings']), 1) + self.assertEqual(body['outings'][0]['title'], 'Ski tour') + + def test_tracks_truncated(self): + with patch('c2corg_api.views.outing_map.TRACKS_LIMIT_MAX', 1): + body = self.app.get( + self._prefix + '?bbox=' + self.bbox, status=200).json + self.assertTrue(body['truncated']) + self.assertEqual(body['outings'], []) diff --git a/c2corg_api/views/outing_map.py b/c2corg_api/views/outing_map.py new file mode 100644 index 000000000..06ce974cb --- /dev/null +++ b/c2corg_api/views/outing_map.py @@ -0,0 +1,177 @@ +import logging + +from cornice.resource import resource, view + +from c2corg_api.caching import cache_outing_map, get_or_create +from c2corg_api.ext.colander_ext import geojson_from_wkbelement +from c2corg_api.models.outing import OUTING_TYPE +from c2corg_api.models.outing_map_queries import ( + get_locales_for_documents, get_outings_heatmap, get_outings_tracks) +from c2corg_api.security.acl import ACLDefault +from c2corg_api.views import cors_policy, get_best_locale +from c2corg_api.views.validation import validate_activities, \ + validate_bbox, validate_preferred_lang_param, validate_zoom + +log = logging.getLogger(__name__) + +# above this number of individual tracks in a bbox, the tracks endpoint +# reports "too many results" instead of returning a partial list +TRACKS_LIMIT_MAX = 100 + +# grid cell size (in meters, EPSG:3857) used to bucket outings for the +# heatmap, indexed by zoom level. The frontend is only expected to call +# this endpoint below the zoom level where it switches to the tracks +# endpoint, so no entries are needed for very high zoom levels. +HEATMAP_CELL_SIZE_BY_ZOOM = { + 0: 40000, + 1: 40000, + 2: 40000, + 3: 30000, + 4: 20000, + 5: 15000, + 6: 10000, + 7: 8000, + 8: 5000, + 9: 3000, + 10: 2000, + 11: 1200, + 12: 700, +} + + +def _cell_size_for_zoom(zoom): + known_zooms = HEATMAP_CELL_SIZE_BY_ZOOM.keys() + clamped_zoom = min(max(zoom, min(known_zooms)), max(known_zooms)) + return HEATMAP_CELL_SIZE_BY_ZOOM[clamped_zoom] + + +def _rounded_bbox_key(bbox, grid=500): + return ','.join(str(int(round(value / grid) * grid)) for value in bbox) + + +def _activities_key(activities): + return ','.join(sorted(activities)) if activities else '' + + +def _titles_by_document_id(document_ids, lang): + locales = get_locales_for_documents(document_ids) + + locales_by_document = {} + for locale in locales: + locales_by_document.setdefault(locale.document_id, {})[ + locale.lang] = locale + + titles = {} + for document_id, available_locales in locales_by_document.items(): + best_locale = get_best_locale(available_locales, lang) + if best_locale: + titles[document_id] = best_locale.title + return titles + + +@resource(path='/outings/map/heatmap', cors_policy=cors_policy) +class OutingsHeatmapRest(ACLDefault): + + @view(validators=[validate_bbox, validate_zoom, validate_activities]) + def get(self): + """Returns a grid of outing-density buckets for the given bbox, + meant to feed a heatmap layer at low/medium map zoom levels. + + Request: + `GET` `/outings/map/heatmap?bbox=...&zoom=...[&act=...]` + + Parameters: + `bbox=xmin,ymin,xmax,ymax` (required, EPSG:3857) + + `zoom=...` (required) + The current map zoom level. Used server-side to pick a grid + cell size, so that a client cannot request an arbitrarily + fine-grained aggregation over a large area. + + `act=...` (optional, comma-separated activity codes) + Restricts the heatmap to the given activities. + + Response: + `{"cell_size": 1200, "buckets": [{"x":.., "y":.., "count":..}]}` + Coordinates are in EPSG:3857, matching the map's projection. + """ + bbox = self.request.validated['bbox'] + zoom = self.request.validated['zoom'] + activities = self.request.validated.get('activities') + cell_size = _cell_size_for_zoom(zoom) + + cache_key = 'heatmap:{}:{}:{}'.format( + _rounded_bbox_key(bbox), _activities_key(activities), cell_size) + + def create(): + rows = get_outings_heatmap(bbox, activities, cell_size) + return { + 'cell_size': cell_size, + 'buckets': [ + {'x': row.x, 'y': row.y, 'count': row.count} + for row in rows + ] + } + + return get_or_create(cache_outing_map, cache_key, create) + + +@resource(path='/outings/map/tracks', cors_policy=cors_policy) +class OutingsTracksRest(ACLDefault): + + @view(validators=[ + validate_bbox, validate_activities, validate_preferred_lang_param]) + def get(self): + """Returns individual outing tracks for the given bbox, meant to + feed a vector layer at high map zoom levels (once the number of + outings in the viewport is bounded). + + Request: + `GET` `/outings/map/tracks?bbox=...[&act=...][&pl=...]` + + Parameters: + `bbox=xmin,ymin,xmax,ymax` (required, EPSG:3857) + + `act=...` (optional, comma-separated activity codes) + + `pl=...` (optional preferred language) + + Response (normal case): + `{"outings": [{"document_id":.., "type": "o", "title":.., + "geometry": {"geom_detail": ""}}], "truncated": false}` + + If more than `TRACKS_LIMIT_MAX` outings intersect the bbox, + an empty, `"truncated": true` response is returned instead of + a silently-incomplete list, so that the frontend can prompt + the user to zoom in further. + """ + bbox = self.request.validated['bbox'] + activities = self.request.validated.get('activities') + lang = self.request.validated.get('lang') + + cache_key = 'tracks:{}:{}:{}'.format( + _rounded_bbox_key(bbox), _activities_key(activities), lang) + + def create(): + rows = get_outings_tracks(bbox, activities, TRACKS_LIMIT_MAX) + if len(rows) > TRACKS_LIMIT_MAX: + return {'outings': [], 'truncated': True} + + titles = _titles_by_document_id( + [row.document_id for row in rows], lang) + + outings = [ + { + 'document_id': row.document_id, + 'type': OUTING_TYPE, + 'title': titles.get(row.document_id), + 'geometry': { + 'geom_detail': + geojson_from_wkbelement(row.geom_detail) + } + } + for row in rows + ] + return {'outings': outings, 'truncated': False} + + return get_or_create(cache_outing_map, cache_key, create) diff --git a/c2corg_api/views/validation.py b/c2corg_api/views/validation.py index 65a8e31c9..2824b3595 100644 --- a/c2corg_api/views/validation.py +++ b/c2corg_api/views/validation.py @@ -19,6 +19,8 @@ from c2corg_api.models.common.associations import valid_associations from c2corg_api.models.common.attributes import default_langs +from c2corg_api.models.common.attributes import activities \ + as valid_activities from c2corg_api.models.common import document_types from colander import null from cornice.errors import Errors @@ -187,6 +189,66 @@ def validate_token_pagination(request, **kwargs): validate_token(request) +def validate_zoom(request, **kwargs): + """Checks that a required `zoom=...` integer query parameter is given. + """ + check_get_for_integer_property(request, 'zoom', True) + + +def validate_bbox(request, **kwargs): + """Checks and parses a required `bbox=xmin,ymin,xmax,ymax` (EPSG:3857) + query parameter into `request.validated['bbox']` as a 4-tuple of + floats. + + This is distinct from `create_bbox_filter` in + `c2corg_api.search.search_filters`, which parses the same shape of + parameter for an Elasticsearch bounding-box filter and reprojects it + to EPSG:4326 in the process. Views that query PostGIS directly want + the bbox to stay in EPSG:3857, matching the SRID of the geometry + columns. + """ + bbox_param = request.GET.get('bbox') + if not bbox_param: + request.errors.add('querystring', 'bbox', 'bbox is missing') + return + + values = bbox_param.split(',') + if len(values) != 4: + request.errors.add('querystring', 'bbox', 'invalid bbox') + return + + try: + xmin, ymin, xmax, ymax = (float(v) for v in values) + except ValueError: + request.errors.add('querystring', 'bbox', 'invalid bbox') + return + + if xmin >= xmax or ymin >= ymax: + request.errors.add('querystring', 'bbox', 'invalid bbox') + return + + request.validated['bbox'] = (xmin, ymin, xmax, ymax) + + +def validate_activities(request, **kwargs): + """Checks and parses an optional `act=activity1,activity2` query + parameter into `request.validated['activities']` (a list, or `None` + if not given). + """ + act_param = request.GET.get('act') + if not act_param: + request.validated['activities'] = None + return + + values = act_param.split(',') + for value in values: + if value not in valid_activities: + request.errors.add('querystring', 'act', 'invalid activity') + return + + request.validated['activities'] = values + + def validate_simple_token_pagination(request, **kwargs): """ Validate token pagination parameters (limit and token) for changes feed.