From a9bb66285a68038da8c3dbc99aee7e321a21336c Mon Sep 17 00:00:00 2001 From: Dustin Lo Date: Thu, 11 Feb 2021 18:48:32 -0800 Subject: [PATCH] Hc 324 (#43) * removed geo.py because its not needed anymore removed get_center function and instead using shapely.geometry centroid to get center not using regex to get the location type fixed test_geonames.py to not use get_center * added get_nearest_cities function for (Multi)Points and (Multi)LineStrings if (Multi)Polygon then use get_cities, else use get_nearest_cities * removed pop_th argument from get_cities set default return size of get_cities to 5 * renamed builder.py -> specs.py (feel its a better name) * removed stuff from test_geonames.py --- grq2/lib/dataset.py | 115 +++++++++++------- grq2/lib/geo.py | 38 ------ grq2/lib/geonames.py | 61 +++++++++- grq2/services/api_v01/service.py | 2 +- .../services/api_v01/{builder.py => specs.py} | 0 grq2/services/api_v02/service.py | 2 +- .../services/api_v02/{builder.py => specs.py} | 0 grq2/services/geonames.py | 3 +- test/output.json | 4 +- test/test_geonames.py | 16 +-- 10 files changed, 142 insertions(+), 99 deletions(-) delete mode 100644 grq2/lib/geo.py rename grq2/services/api_v01/{builder.py => specs.py} (100%) rename grq2/services/api_v02/{builder.py => specs.py} (100%) diff --git a/grq2/lib/dataset.py b/grq2/lib/dataset.py index bb7e3be..3b34b43 100644 --- a/grq2/lib/dataset.py +++ b/grq2/lib/dataset.py @@ -4,20 +4,56 @@ from __future__ import absolute_import from builtins import str from future import standard_library -standard_library.install_aliases() import json import traceback -import re + +from shapely.geometry import shape from grq2 import app, grq_es -from grq2.lib.geonames import get_cities, get_continents -from grq2.lib.geo import get_center +from grq2.lib.geonames import get_cities, get_nearest_cities, get_continents from grq2.lib.time_utils import getTemporalSpanInDays as get_ts +standard_library.install_aliases() + -POLYGON_RE = re.compile(r'^polygon$', re.I) -MULTIPOLYGON_RE = re.compile(r'^multipolygon$', re.I) -LINESTRING_RE = re.compile(r'^linestring$', re.I) +_POINT = 'Point' +_MULTIPOINT = 'MultiPoint' +_LINESTRING = 'LineString' +_MULTILINESTRING = 'MultiLineString' +_POLYGON = 'Polygon' +_MULTIPOLYGON = 'MultiPolygon' + +GEOJSON_TYPES = {_POINT, _MULTIPOINT, _LINESTRING, _MULTILINESTRING, _POLYGON, _MULTIPOLYGON} + + +def map_geojson_type(geo_type): + """ + maps the GEOJson type to the correct naming + GEOJson types are case sensitive and can be these 6 types: + 'Point', 'MultiPoint', 'LineString', 'MultiLineString', 'Polygon', 'MultiPolygon' + :param geo_type: str, GEOJson type + :return: str + """ + # TODO: ES7.1's geo_shape doesn't support Point & MultiPoint but 7.7+ does, will need to revisit later + + if geo_type in GEOJSON_TYPES: + return geo_type + + geo_type_lower = geo_type.lower() + if _POINT.lower() == geo_type_lower: + return _POINT + elif _MULTIPOINT.lower() == geo_type_lower: + return _MULTIPOINT + elif _MULTILINESTRING.lower() == geo_type_lower: + return _MULTILINESTRING + elif _LINESTRING.lower() == geo_type_lower: + return _LINESTRING + elif _MULTIPOLYGON.lower() == geo_type_lower: + return _MULTIPOLYGON + elif _POLYGON.lower() == geo_type_lower: + return _POLYGON + else: + return None def update(update_json): @@ -41,56 +77,47 @@ def update(update_json): # add reverse geo-location data if 'location' in update_json: - # get coords and if it's a multipolygon - loc_type = update_json['location']['type'] - mp = False - if POLYGON_RE.search(loc_type): - coords = update_json['location']['coordinates'][0] - elif MULTIPOLYGON_RE.search(loc_type): - coords = update_json['location']['coordinates'][0] - mp = True - elif LINESTRING_RE.search(loc_type): - coords = update_json['location']['coordinates'] - else: - # TODO: NEED TO HANDLE DEFAULT CASE (MAYBE POINT?) - pass + location = {**update_json['location']} # copying location to be used to create a shapely geometry object + loc_type = location['type'] - # add cities - update_json['city'] = get_cities(coords, pop_th=0, multipolygon=mp) + geo_json_type = map_geojson_type(loc_type) + location['type'] = geo_json_type # setting proper GEOJson type, ex. multipolygon -> MultiPolygon + update_json['location']['type'] = geo_json_type.lower() # add center if missing if 'center' not in update_json: - if mp: - center_lon, center_lat = get_center(coords[0]) - update_json['center'] = { - 'type': 'point', - 'coordinates': [center_lon, center_lat] - } - else: - center_lon, center_lat = get_center(coords) - update_json['center'] = { - 'type': 'point', - 'coordinates': [center_lon, center_lat] - } + geo_shape = shape(location) + centroid = geo_shape.centroid + update_json['center'] = { + 'type': 'point', + 'coordinates': [centroid.x, centroid.y] + } + + # extract coordinates from center + lon, lat = update_json['center']['coordinates'] + + # add cities + if geo_json_type in (_POLYGON, _MULTIPOLYGON): + mp = True if geo_json_type == _MULTIPOLYGON else False + coords = location['coordinates'][0] + update_json['city'] = get_cities(coords, multipolygon=mp) + elif geo_json_type in (_POINT, _MULTIPOINT, _LINESTRING, _MULTILINESTRING): + update_json['city'] = get_nearest_cities(lon, lat) + else: + raise TypeError('%s is not a valid GEOJson type (or un-supported): %s' % (geo_json_type, GEOJSON_TYPES)) # add closest continent - lon, lat = update_json['center']['coordinates'] continents = get_continents(lon, lat) - update_json['continent'] = continents[0]['name'] if len( - continents) > 0 else None + update_json['continent'] = continents[0]['name'] if len(continents) > 0 else None # set temporal_span - if update_json.get('starttime', None) is not None and \ - update_json.get('endtime', None) is not None: - - if isinstance(update_json['starttime'], str) and \ - isinstance(update_json['endtime'], str): - + if update_json.get('starttime', None) is not None and update_json.get('endtime', None) is not None: + if isinstance(update_json['starttime'], str) and isinstance(update_json['endtime'], str): start_time = update_json['starttime'] end_time = update_json['endtime'] update_json['temporal_span'] = get_ts(start_time, end_time) - result = grq_es.index_document(index=index, body=update_json, id=update_json['id']) # indexing to ES + result = grq_es.index_document(index=index, body=update_json, id=update_json['id']) app.logger.debug("%s" % json.dumps(result, indent=2)) # update custom aliases (Fixing HC-23) diff --git a/grq2/lib/geo.py b/grq2/lib/geo.py deleted file mode 100644 index 1156875..0000000 --- a/grq2/lib/geo.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import unicode_literals -from __future__ import print_function -from __future__ import division -from __future__ import absolute_import -from future import standard_library -standard_library.install_aliases() -import json -import requests -import types -from pprint import pformat -from shapely.geometry import Polygon, MultiPolygon -import shapely.wkt -import cartopy.crs as ccrs - - -from grq2 import app - - -def get_center(bbox): - """ - Return center (lon, lat) of bbox. - """ - - poly = Polygon(bbox) - src_proj = ccrs.TransverseMercator() - tgt_proj = src_proj - for point in bbox: - if point[0] == -180. or point[0] == 180.: - if point[1] > 0: - tgt_proj = ccrs.RotatedPole(0., 90.) - else: - tgt_proj = ccrs.RotatedPole(0., -90.) - break - multipolygons = tgt_proj.project_geometry(poly, src_proj) - multipolygons = multipolygons.simplify(10.) - center_lon = multipolygons.centroid.x - center_lat = multipolygons.centroid.y - return center_lon, center_lat diff --git a/grq2/lib/geonames.py b/grq2/lib/geonames.py index 2978913..9dc6d39 100644 --- a/grq2/lib/geonames.py +++ b/grq2/lib/geonames.py @@ -9,7 +9,7 @@ from grq2 import app, grq_es -def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False): +def get_cities(polygon, size=5, multipolygon=False): """ Spatial search of top populated cities within a bounding box. @@ -74,7 +74,7 @@ def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False): { "range": { "population": { - "gte": 0 + "gte": 1 } } } @@ -109,7 +109,60 @@ def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False): index = app.config['GEONAMES_INDEX'] res = grq_es.search(index=index, body=query) # query for results - app.logger.debug("get_cities(): %s" % json.dumps(query, indent=2)) + app.logger.debug("get_cities(): %s" % json.dumps(query)) + + results = [] + for hit in res['hits']['hits']: + results.append(hit['_source']) + return results + + +def get_nearest_cities(lon, lat, size=5): + """ + :param lon: float lon of center (ex. -122.61067217547183) + :param lat: float lat of center (ex. 40.6046338643702) + :param size: return size of results + :return: List[Dict] + """ + query = { + "size": size, + "sort": [ + { + "_geo_distance": { + "location": [lon, lat], + "order": "asc", + "unit": "km" + } + } + ], + "query": { + "bool": { + "filter": [ + { + "term": { + "feature_class": "P" + } + }, + { + "range": { + "population": { + "gte": 1 + } + } + }, + { + "geo_distance": { + "distance": "100km", + "location": [lon, lat] + } + } + ] + } + } + } + index = app.config['GEONAMES_INDEX'] # query for results + res = grq_es.search(index=index, body=query) + app.logger.debug("get_continents(): %s" % json.dumps(query, indent=2)) results = [] for hit in res['hits']['hits']: @@ -120,6 +173,8 @@ def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False): def get_continents(lon, lat): """ Spatial search of closest continents to the specified geo point. + :param lon: float lon of center (ex. -122.61067217547183) + :param lat: float lat of center (ex. 40.6046338643702) Example query DSL: { diff --git a/grq2/services/api_v01/service.py b/grq2/services/api_v01/service.py index 3875d77..24614b3 100644 --- a/grq2/services/api_v01/service.py +++ b/grq2/services/api_v01/service.py @@ -9,7 +9,7 @@ from flask import Blueprint from flask_restx import Api, apidoc, Namespace -from .builder import hysds_io_ns +from .specs import hysds_io_ns services = Blueprint('api_v0-1', __name__, url_prefix='/api/v0.1') diff --git a/grq2/services/api_v01/builder.py b/grq2/services/api_v01/specs.py similarity index 100% rename from grq2/services/api_v01/builder.py rename to grq2/services/api_v01/specs.py diff --git a/grq2/services/api_v02/service.py b/grq2/services/api_v02/service.py index ec9f598..2344921 100644 --- a/grq2/services/api_v02/service.py +++ b/grq2/services/api_v02/service.py @@ -9,7 +9,7 @@ from flask import Blueprint from flask_restx import Api, apidoc, Namespace -from .builder import hysds_io_ns +from .specs import hysds_io_ns services = Blueprint('api_v0-2', __name__, url_prefix='/api/v0.2') diff --git a/grq2/services/api_v02/builder.py b/grq2/services/api_v02/specs.py similarity index 100% rename from grq2/services/api_v02/builder.py rename to grq2/services/api_v02/specs.py diff --git a/grq2/services/geonames.py b/grq2/services/geonames.py index db0a99d..31e93c4 100644 --- a/grq2/services/geonames.py +++ b/grq2/services/geonames.py @@ -31,7 +31,6 @@ def cities(): # get query polygon = request.args.get('polygon', None) - pop_th = int(request.args.get('population', 1000000)) size = int(request.args.get('size', 10)) # if no polygon passed, show help @@ -55,7 +54,7 @@ def cities(): # get results try: - cities = get_cities(polygon, pop_th, size) + cities = get_cities(polygon, size) except Exception as e: return jsonify({ 'success': False, diff --git a/test/output.json b/test/output.json index c799e64..077614b 100644 --- a/test/output.json +++ b/test/output.json @@ -29,7 +29,7 @@ "id": "index-airs.aqua_cloudsat-v4.0-2006.06.18.045", "metadata": { "center": { - "type": "point", + "type": "point", "coordinates": [ 140.1861572265625, -19.95725440979004 @@ -40,7 +40,7 @@ "level": "L2", "version": "4.0", "location": { - "type": "linestring", + "type": "linestring", "coordinates": [ [ 142.86524963378906, diff --git a/test/test_geonames.py b/test/test_geonames.py index b0051c9..98434f1 100755 --- a/test/test_geonames.py +++ b/test/test_geonames.py @@ -6,30 +6,30 @@ from builtins import open from future import standard_library standard_library.install_aliases() -import os -import sys + import json -from pprint import pprint from grq2.lib.geonames import get_cities, get_continents -from grq2.lib.geo import get_center def main(): # test linestring with open('output.json') as f: m = json.load(f) - coords = m['metadata']['location']['coordinates'] + + location = m['metadata']['location'] + coords = location['coordinates'] + lon, lat = m['metadata']['center']['coordinates'] - cities = get_cities(coords, pop_th=0) + + # cities = get_cities(coords) continents = get_continents(lon, lat) + if len(continents) > 0: continent = continents[0]['name'] else: continent = None - pprint(cities) print(continent) - print((get_center(coords))) if __name__ == "__main__":