Hc 324 (#43)

* removed geo.py because its not needed anymore

removed get_center function and instead using shapely.geometry centroid to get center
not using regex to get the location type
fixed test_geonames.py to not use get_center

* added get_nearest_cities function for (Multi)Points and (Multi)LineStrings

if (Multi)Polygon then use get_cities, else use get_nearest_cities

* removed pop_th argument from get_cities

set default return size of get_cities to 5

* renamed builder.py -> specs.py (feel its a better name)

* removed stuff from test_geonames.py

grq2/lib/dataset.py

@@ -4,20 +4,56 @@
 from __future__ import absolute_import
 from builtins import str
 from future import standard_library
-standard_library.install_aliases()
 
 import json
 import traceback
-import re
+
+from shapely.geometry import shape
 
 from grq2 import app, grq_es
-from grq2.lib.geonames import get_cities, get_continents
-from grq2.lib.geo import get_center
+from grq2.lib.geonames import get_cities, get_nearest_cities, get_continents
 from grq2.lib.time_utils import getTemporalSpanInDays as get_ts
+standard_library.install_aliases()
+
 
-POLYGON_RE = re.compile(r'^polygon$', re.I)
-MULTIPOLYGON_RE = re.compile(r'^multipolygon$', re.I)
-LINESTRING_RE = re.compile(r'^linestring$', re.I)
+_POINT = 'Point'
+_MULTIPOINT = 'MultiPoint'
+_LINESTRING = 'LineString'
+_MULTILINESTRING = 'MultiLineString'
+_POLYGON = 'Polygon'
+_MULTIPOLYGON = 'MultiPolygon'
+
+GEOJSON_TYPES = {_POINT, _MULTIPOINT, _LINESTRING, _MULTILINESTRING, _POLYGON, _MULTIPOLYGON}
+
+
+def map_geojson_type(geo_type):
+    """
+    maps the GEOJson type to the correct naming
+    GEOJson types are case sensitive and can be these 6 types:
+        'Point', 'MultiPoint', 'LineString', 'MultiLineString', 'Polygon', 'MultiPolygon'
+    :param geo_type: str, GEOJson type
+    :return: str
+    """
+    # TODO: ES7.1's geo_shape doesn't support Point & MultiPoint but 7.7+ does, will need to revisit later
+
+    if geo_type in GEOJSON_TYPES:
+        return geo_type
+
+    geo_type_lower = geo_type.lower()
+    if _POINT.lower() == geo_type_lower:
+        return _POINT
+    elif _MULTIPOINT.lower() == geo_type_lower:
+        return _MULTIPOINT
+    elif _MULTILINESTRING.lower() == geo_type_lower:
+        return _MULTILINESTRING
+    elif _LINESTRING.lower() == geo_type_lower:
+        return _LINESTRING
+    elif _MULTIPOLYGON.lower() == geo_type_lower:
+        return _MULTIPOLYGON
+    elif _POLYGON.lower() == geo_type_lower:
+        return _POLYGON
+    else:
+        return None
 
 
 def update(update_json):
@@ -41,56 +77,47 @@ def update(update_json):
 
     # add reverse geo-location data
     if 'location' in update_json:
-        # get coords and if it's a multipolygon
-        loc_type = update_json['location']['type']
-        mp = False
-        if POLYGON_RE.search(loc_type):
-            coords = update_json['location']['coordinates'][0]
-        elif MULTIPOLYGON_RE.search(loc_type):
-            coords = update_json['location']['coordinates'][0]
-            mp = True
-        elif LINESTRING_RE.search(loc_type):
-            coords = update_json['location']['coordinates']
-        else:
-            # TODO: NEED TO HANDLE DEFAULT CASE (MAYBE POINT?)
-            pass
+        location = {**update_json['location']}  # copying location to be used to create a shapely geometry object
+        loc_type = location['type']
 
-        # add cities
-        update_json['city'] = get_cities(coords, pop_th=0, multipolygon=mp)
+        geo_json_type = map_geojson_type(loc_type)
+        location['type'] = geo_json_type  # setting proper GEOJson type, ex. multipolygon -> MultiPolygon
+        update_json['location']['type'] = geo_json_type.lower()
 
         # add center if missing
         if 'center' not in update_json:
-            if mp:
-                center_lon, center_lat = get_center(coords[0])
-                update_json['center'] = {
-                    'type': 'point',
-                    'coordinates': [center_lon, center_lat]
-                }
-            else:
-                center_lon, center_lat = get_center(coords)
-                update_json['center'] = {
-                    'type': 'point',
-                    'coordinates': [center_lon, center_lat]
-                }
+            geo_shape = shape(location)
+            centroid = geo_shape.centroid
+            update_json['center'] = {
+                'type': 'point',
+                'coordinates': [centroid.x, centroid.y]
+            }
+
+        # extract coordinates from center
+        lon, lat = update_json['center']['coordinates']
+
+        # add cities
+        if geo_json_type in (_POLYGON, _MULTIPOLYGON):
+            mp = True if geo_json_type == _MULTIPOLYGON else False
+            coords = location['coordinates'][0]
+            update_json['city'] = get_cities(coords, multipolygon=mp)
+        elif geo_json_type in (_POINT, _MULTIPOINT, _LINESTRING, _MULTILINESTRING):
+            update_json['city'] = get_nearest_cities(lon, lat)
+        else:
+            raise TypeError('%s is not a valid GEOJson type (or un-supported): %s' % (geo_json_type, GEOJSON_TYPES))
 
         # add closest continent
-        lon, lat = update_json['center']['coordinates']
         continents = get_continents(lon, lat)
-        update_json['continent'] = continents[0]['name'] if len(
-            continents) > 0 else None
+        update_json['continent'] = continents[0]['name'] if len(continents) > 0 else None
 
     # set temporal_span
-    if update_json.get('starttime', None) is not None and \
-       update_json.get('endtime', None) is not None:
-
-        if isinstance(update_json['starttime'], str) and \
-           isinstance(update_json['endtime'], str):
-
+    if update_json.get('starttime', None) is not None and update_json.get('endtime', None) is not None:
+        if isinstance(update_json['starttime'], str) and isinstance(update_json['endtime'], str):
             start_time = update_json['starttime']
             end_time = update_json['endtime']
             update_json['temporal_span'] = get_ts(start_time, end_time)
 
-    result = grq_es.index_document(index=index, body=update_json, id=update_json['id'])  # indexing to ES
+    result = grq_es.index_document(index=index, body=update_json, id=update_json['id'])
     app.logger.debug("%s" % json.dumps(result, indent=2))
 
     # update custom aliases (Fixing HC-23)

grq2/lib/geonames.py

@@ -9,7 +9,7 @@
 from grq2 import app, grq_es
 
 
-def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False):
+def get_cities(polygon, size=5, multipolygon=False):
     """
     Spatial search of top populated cities within a bounding box.
 
@@ -74,7 +74,7 @@ def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False):
                     {
                         "range": {
                             "population": {
-                                "gte": 0
+                                "gte": 1
                             }
                         }
                     }
@@ -109,7 +109,60 @@ def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False):
 
     index = app.config['GEONAMES_INDEX']
     res = grq_es.search(index=index, body=query)  # query for results
-    app.logger.debug("get_cities(): %s" % json.dumps(query, indent=2))
+    app.logger.debug("get_cities(): %s" % json.dumps(query))
+
+    results = []
+    for hit in res['hits']['hits']:
+        results.append(hit['_source'])
+    return results
+
+
+def get_nearest_cities(lon, lat, size=5):
+    """
+    :param lon: float lon of center (ex. -122.61067217547183)
+    :param lat: float lat of center (ex. 40.6046338643702)
+    :param size: return size of results
+    :return: List[Dict]
+    """
+    query = {
+        "size": size,
+        "sort": [
+            {
+                "_geo_distance": {
+                    "location": [lon, lat],
+                    "order": "asc",
+                    "unit": "km"
+                }
+            }
+        ],
+        "query": {
+            "bool": {
+                "filter": [
+                    {
+                        "term": {
+                            "feature_class": "P"
+                        }
+                    },
+                    {
+                        "range": {
+                            "population": {
+                                "gte": 1
+                            }
+                        }
+                    },
+                    {
+                        "geo_distance": {
+                            "distance": "100km",
+                            "location": [lon, lat]
+                        }
+                    }
+                ]
+            }
+        }
+    }
+    index = app.config['GEONAMES_INDEX']  # query for results
+    res = grq_es.search(index=index, body=query)
+    app.logger.debug("get_continents(): %s" % json.dumps(query, indent=2))
 
     results = []
     for hit in res['hits']['hits']:
@@ -120,6 +173,8 @@ def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False):
 def get_continents(lon, lat):
     """
     Spatial search of closest continents to the specified geo point.
+    :param lon: float lon of center (ex. -122.61067217547183)
+    :param lat: float lat of center (ex. 40.6046338643702)
 
     Example query DSL:
     {

grq2/services/api_v01/service.py

@@ -9,7 +9,7 @@
 from flask import Blueprint
 from flask_restx import Api, apidoc, Namespace
 
-from .builder import hysds_io_ns
+from .specs import hysds_io_ns
 
 
 services = Blueprint('api_v0-1', __name__, url_prefix='/api/v0.1')

grq2/services/api_v02/service.py

@@ -9,7 +9,7 @@
 from flask import Blueprint
 from flask_restx import Api, apidoc, Namespace
 
-from .builder import hysds_io_ns
+from .specs import hysds_io_ns
 
 
 services = Blueprint('api_v0-2', __name__, url_prefix='/api/v0.2')

grq2/services/geonames.py

@@ -31,7 +31,6 @@ def cities():
 
     # get query
     polygon = request.args.get('polygon', None)
-    pop_th = int(request.args.get('population', 1000000))
     size = int(request.args.get('size', 10))
 
     # if no polygon passed, show help
@@ -55,7 +54,7 @@ def cities():
 
     # get results
     try:
-        cities = get_cities(polygon, pop_th, size)
+        cities = get_cities(polygon, size)
     except Exception as e:
         return jsonify({
             'success': False,

test/output.json

@@ -29,7 +29,7 @@
   "id": "index-airs.aqua_cloudsat-v4.0-2006.06.18.045", 
   "metadata": {
     "center": {
-      "type": "point", 
+      "type": "point",
       "coordinates": [
         140.1861572265625, 
         -19.95725440979004
@@ -40,7 +40,7 @@
     "level": "L2", 
     "version": "4.0", 
     "location": {
-      "type": "linestring", 
+      "type": "linestring",
       "coordinates": [
         [
           142.86524963378906, 

test/test_geonames.py

@@ -6,30 +6,30 @@
 from builtins import open
 from future import standard_library
 standard_library.install_aliases()
-import os
-import sys
+
 import json
-from pprint import pprint
 
 from grq2.lib.geonames import get_cities, get_continents
-from grq2.lib.geo import get_center
 
 
 def main():
     # test linestring
     with open('output.json') as f:
         m = json.load(f)
-    coords = m['metadata']['location']['coordinates']
+
+    location = m['metadata']['location']
+    coords = location['coordinates']
+
     lon, lat = m['metadata']['center']['coordinates']
-    cities = get_cities(coords, pop_th=0)
+
+    # cities = get_cities(coords)
     continents = get_continents(lon, lat)
+
     if len(continents) > 0:
         continent = continents[0]['name']
     else:
         continent = None
-    pprint(cities)
     print(continent)
-    print((get_center(coords)))
 
 
 if __name__ == "__main__":