From 482b293a1d205a3b60b45128c10b12b70ddb135f Mon Sep 17 00:00:00 2001 From: Gerald Manipon Date: Wed, 20 Mar 2019 08:21:42 -0700 Subject: [PATCH] python3 port (#4) * port to python3 (#3) * initial 2to3 run * remove supervisor from required (not python3 compatible yet) * format to PEP8 spec * initial 2to3 run; update dependencies * remediate CVE-2015-8857 and CVE-2015-8858 * pasteurize * remove imports not yet installed via install_requires * require future * bump version --- grq2/__init__.py | 9 ++ grq2/lib/dataset.py | 35 +++-- grq2/lib/geo.py | 16 ++- grq2/lib/geonames.py | 18 ++- grq2/lib/time_utils.py | 72 ++++++---- grq2/lib/utils.py | 11 +- grq2/services/api_v01.py | 128 ++++++++++------- grq2/services/api_v02.py | 13 +- grq2/services/geonames.py | 16 ++- grq2/services/main.py | 7 + grq2/services/query.py | 170 ++++++++++++++--------- grq2/static/Leaflet.draw/package.json | 2 +- grq2/views/main.py | 7 + run.py | 8 +- run_debug.py | 8 +- scripts/add_io_users.py | 66 ++++++--- scripts/clean_out_ios.py | 37 +++-- scripts/copy_add_groups_metadata.py | 32 +++-- scripts/copy_add_new_metadata.py | 34 +++-- scripts/copy_add_resolutions_airx3ret.py | 34 +++-- scripts/copy_add_resolutions_airx3spd.py | 34 +++-- scripts/copy_index.py | 29 ++-- scripts/copy_modify_index.py | 29 ++-- scripts/copy_user_rules.py | 34 +++-- scripts/copy_user_tags.py | 32 +++-- scripts/create_index.py | 10 +- scripts/delete_s3_objects.py | 11 +- scripts/fix_temporal_span.py | 99 +++++++------ scripts/geonames/import_ES.py | 117 +++++++++------- scripts/install_es_template.py | 16 ++- scripts/migrate_s3_datasets.py | 56 +++++--- scripts/rename_s3_datasets.py | 101 +++++++++----- scripts/republish_s3_products.py | 31 +++-- scripts/rm_io_users.py | 66 ++++++--- setup.py | 12 +- test/test_geonames.py | 17 ++- 36 files changed, 951 insertions(+), 466 deletions(-) diff --git a/grq2/__init__.py b/grq2/__init__.py index 1fe594c..15c1d66 100644 --- a/grq2/__init__.py +++ b/grq2/__init__.py @@ -1,3 +1,9 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from flask import Flask app = Flask(__name__) @@ -10,13 +16,16 @@ # services blueprints from grq2.services.main import mod as mainModule app.register_blueprint(mainModule) + from grq2.services.query import mod as queryModule app.register_blueprint(queryModule) + from grq2.services.geonames import mod as geonamesModule app.register_blueprint(geonamesModule) # rest API blueprints from grq2.services.api_v01 import services as api_v01Services app.register_blueprint(api_v01Services) + from grq2.services.api_v02 import services as api_v02Services app.register_blueprint(api_v02Services) diff --git a/grq2/lib/dataset.py b/grq2/lib/dataset.py index 140f2d4..67a2797 100644 --- a/grq2/lib/dataset.py +++ b/grq2/lib/dataset.py @@ -1,4 +1,15 @@ -import json, traceback, re, requests, types +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import str +from future import standard_library +standard_library.install_aliases() +import json +import traceback +import re +import requests +import types from flask import jsonify, Blueprint, request from pprint import pformat from elasticsearch import Elasticsearch @@ -78,24 +89,28 @@ def update(update_json): # add closest continent lon, lat = update_json['center']['coordinates'] continents = get_continents(lon, lat) - update_json['continent'] = continents[0]['name'] if len(continents) > 0 else None + update_json['continent'] = continents[0]['name'] if len( + continents) > 0 else None # set temporal_span if update_json.get('starttime', None) is not None and \ update_json.get('endtime', None) is not None: - if isinstance(update_json['starttime'], types.StringTypes) and \ - isinstance(update_json['endtime'], types.StringTypes): - update_json['temporal_span'] = get_ts(update_json['starttime'], update_json['endtime']) - + if isinstance(update_json['starttime'], str) and \ + isinstance(update_json['endtime'], str): + update_json['temporal_span'] = get_ts( + update_json['starttime'], update_json['endtime']) + #app.logger.debug("update_json:\n%s" % json.dumps(update_json, indent=2)) # update in elasticsearch try: es = Elasticsearch(hosts=[app.config['ES_URL']]) - ret = es.index(index=index, doc_type=doctype, id=update_json['id'], body=update_json) - except Exception, e: - message = "Got exception trying to index dataset: %s\n%s" % (str(e), traceback.format_exc()) + ret = es.index(index=index, doc_type=doctype, + id=update_json['id'], body=update_json) + except Exception as e: + message = "Got exception trying to index dataset: %s\n%s" % ( + str(e), traceback.format_exc()) app.logger.debug(message) return jsonify({ 'success': False, @@ -116,7 +131,7 @@ def update(update_json): "actions": actions }) #app.logger.debug("alias_ret: %s" % json.dumps(alias_ret, indent=2)) - except Exception, e: + except Exception as e: app.logger.debug("Got exception trying to add aliases to index: %s\n%s\nContinuing on." % (str(e), traceback.format_exc())) diff --git a/grq2/lib/geo.py b/grq2/lib/geo.py index 08bf517..1156875 100644 --- a/grq2/lib/geo.py +++ b/grq2/lib/geo.py @@ -1,4 +1,12 @@ -import json, requests, types +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import types from pprint import pformat from shapely.geometry import Polygon, MultiPolygon import shapely.wkt @@ -18,8 +26,10 @@ def get_center(bbox): tgt_proj = src_proj for point in bbox: if point[0] == -180. or point[0] == 180.: - if point[1] > 0: tgt_proj = ccrs.RotatedPole(0., 90.) - else: tgt_proj = ccrs.RotatedPole(0., -90.) + if point[1] > 0: + tgt_proj = ccrs.RotatedPole(0., 90.) + else: + tgt_proj = ccrs.RotatedPole(0., -90.) break multipolygons = tgt_proj.project_geometry(poly, src_proj) multipolygons = multipolygons.simplify(10.) diff --git a/grq2/lib/geonames.py b/grq2/lib/geonames.py index 35e3de7..fc79741 100644 --- a/grq2/lib/geonames.py +++ b/grq2/lib/geonames.py @@ -1,4 +1,12 @@ -import json, requests, types +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import types from pprint import pformat from grq2 import app @@ -113,7 +121,8 @@ def get_cities(polygon, pop_th=1000000, size=20, multipolygon=False): # query for results es_url = app.config['ES_URL'] index = app.config['GEONAMES_INDEX'] - r = requests.post('%s/%s/_search' % (es_url, index), data=json.dumps(query)) + r = requests.post('%s/%s/_search' % + (es_url, index), data=json.dumps(query)) app.logger.debug("get_cities(): %s" % json.dumps(query, indent=2)) if r.status_code != 200: raise RuntimeError("Failed to get cities: %s" % pformat(r.json())) @@ -181,7 +190,7 @@ def get_continents(lon, lat): "sort": [ { "_geo_distance": { - "location": [ lon, lat ], + "location": [lon, lat], "order": "asc", "unit": "km" } @@ -195,7 +204,8 @@ def get_continents(lon, lat): # query for results es_url = app.config['ES_URL'] index = app.config['GEONAMES_INDEX'] - r = requests.post('%s/%s/_search' % (es_url, index), data=json.dumps(query)) + r = requests.post('%s/%s/_search' % + (es_url, index), data=json.dumps(query)) app.logger.debug("get_continents(): %s" % json.dumps(query, indent=2)) if r.status_code != 200: raise RuntimeError("Failed to get cities: %s" % pformat(r.json())) diff --git a/grq2/lib/time_utils.py b/grq2/lib/time_utils.py index 4de41cd..9757a36 100644 --- a/grq2/lib/time_utils.py +++ b/grq2/lib/time_utils.py @@ -1,11 +1,21 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import map +from future import standard_library +standard_library.install_aliases() +import time as _time from datetime import tzinfo, timedelta, datetime -import re, time +import re +import time ZERO = timedelta(0) HOUR = timedelta(hours=1) # A UTC class. + class UTC(tzinfo): """UTC""" @@ -18,17 +28,19 @@ def tzname(self, dt): def dst(self, dt): return ZERO + utc = UTC() # A class building tzinfo objects for fixed-offset time zones. # Note that FixedOffset(0, "UTC") is a different way to build a # UTC tzinfo object. + class FixedOffset(tzinfo): """Fixed offset in minutes east from UTC.""" def __init__(self, offset, name): - self.__offset = timedelta(minutes = offset) + self.__offset = timedelta(minutes=offset) self.__name = name def utcoffset(self, dt): @@ -42,16 +54,16 @@ def dst(self, dt): # A class capturing the platform's idea of local time. -import time as _time -STDOFFSET = timedelta(seconds = -_time.timezone) +STDOFFSET = timedelta(seconds=-_time.timezone) if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) + DSTOFFSET = timedelta(seconds=-_time.altzone) else: DSTOFFSET = STDOFFSET DSTDIFF = DSTOFFSET - STDOFFSET + class LocalTimezone(tzinfo): def utcoffset(self, dt): @@ -77,6 +89,7 @@ def _isdst(self, dt): tt = _time.localtime(stamp) return tt.tm_isdst > 0 + Local = LocalTimezone() @@ -114,6 +127,7 @@ def first_sunday_on_or_after(dt): DSTSTART_1967_1986 = datetime(1, 4, 24, 2) DSTEND_1967_1986 = DSTEND_1987_2006 + class USTimeZone(tzinfo): def __init__(self, hours, reprname, stdname, dstname): @@ -164,50 +178,60 @@ def dst(self, dt): else: return ZERO -Eastern = USTimeZone(-5, "Eastern", "EST", "EDT") -Central = USTimeZone(-6, "Central", "CST", "CDT") + +Eastern = USTimeZone(-5, "Eastern", "EST", "EDT") +Central = USTimeZone(-6, "Central", "CST", "CDT") Mountain = USTimeZone(-7, "Mountain", "MST", "MDT") -Pacific = USTimeZone(-8, "Pacific", "PST", "PDT") +Pacific = USTimeZone(-8, "Pacific", "PST", "PDT") + def getFormattedDate(dt): """Return formatted date string.""" - + return dt.strftime("%A, %B %d %Y %I:%M%p") + def getPSTFromUTC(dt): """Return PST datetime object from unaware UTC datetime object.""" - + return dt.replace(tzinfo=utc).astimezone(Pacific) + def getMDY(t): """Return MMM DD, YYYY.""" - + tm = time.strptime(t, "%Y-%m-%d %H:%M:%S") return time.strftime("%b %d, %Y", tm) + def getTimeElementsFromString(dtStr): - """Return tuple of (year,month,day,hour,minute,second) from date time string.""" - - match = re.match(r'^(\d{4})[/-](\d{2})[/-](\d{2})[\s*T](\d{2}):(\d{2}):(\d{2})(?:\.\d+)?Z?$',dtStr) - if match: (year,month,day,hour,minute,second) = map(int,match.groups()) - else: - match = re.match(r'^(\d{4})[/-](\d{2})[/-](\d{2})$',dtStr) - if match: - (year,month,day) = map(int,match.groups()) - (hour,minute,second) = (0,0,0) - else: raise(RuntimeError("Failed to recognize date format: %s" % dtStr)) - return (year,month,day,hour,minute,second) + """Return tuple of (year,month,day,hour,minute,second) from date time string.""" + + match = re.match( + r'^(\d{4})[/-](\d{2})[/-](\d{2})[\s*T](\d{2}):(\d{2}):(\d{2})(?:\.\d+)?Z?$', dtStr) + if match: + (year, month, day, hour, minute, second) = list(map(int, match.groups())) + else: + match = re.match(r'^(\d{4})[/-](\d{2})[/-](\d{2})$', dtStr) + if match: + (year, month, day) = list(map(int, match.groups())) + (hour, minute, second) = (0, 0, 0) + else: + raise RuntimeError + return (year, month, day, hour, minute, second) + def getDatetimeFromString(dtStr, dayOnly=False): """Return datetime object from date time string.""" - - (year,month,day,hour,minute,second) = getTimeElementsFromString(dtStr) + + (year, month, day, hour, minute, second) = getTimeElementsFromString(dtStr) if dayOnly: return datetime(year=year, month=month, day=day) else: return datetime(year=year, month=month, day=day, hour=hour, minute=minute, second=second) + def getTemporalSpanInDays(dt1, dt2): """Return temporal timespan in days.""" diff --git a/grq2/lib/utils.py b/grq2/lib/utils.py index d4c36c0..ef3433e 100644 --- a/grq2/lib/utils.py +++ b/grq2/lib/utils.py @@ -1,4 +1,13 @@ -import os, sys, json +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import open +from future import standard_library +standard_library.install_aliases() +import os +import sys +import json from grq2 import app diff --git a/grq2/services/api_v01.py b/grq2/services/api_v01.py index dd2e3f6..f225dc8 100644 --- a/grq2/services/api_v01.py +++ b/grq2/services/api_v01.py @@ -1,4 +1,17 @@ -import os, sys, json, requests, types, re, traceback +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import str +from future import standard_library +standard_library.install_aliases() +import os +import sys +import json +import requests +import types +import re +import traceback from flask import jsonify, Blueprint, request, Response, render_template, make_response from flask_restplus import Api, apidoc, Resource, fields @@ -17,7 +30,8 @@ ns = api.namespace(NAMESPACE, description="GRQ operations") HYSDS_IO_NS = "hysds_io" -hysds_io_ns = api.namespace(HYSDS_IO_NS, description="HySDS IO operations") +hysds_io_ns = api.namespace(HYSDS_IO_NS, description="HySDS IO operations") + @services.route('/doc/', endpoint='api_doc') def swagger_ui(): @@ -25,8 +39,8 @@ def swagger_ui(): @ns.route('/dataset/index', endpoint='dataset_index') -@api.doc(responses={ 200: "Success", - 500: "Execution failed" }, +@api.doc(responses={200: "Success", + 500: "Execution failed"}, description="Dataset index.") class IndexDataset(Resource): """Dataset indexing API.""" @@ -50,27 +64,32 @@ class IndexDataset(Resource): def post(self): # get info - info = request.form.get('dataset_info', request.args.get('dataset_info', None)) + info = request.form.get( + 'dataset_info', request.args.get('dataset_info', None)) if info is not None: - try: info = json.loads(info) - except Exception, e: + try: + info = json.loads(info) + except Exception as e: message = "Failed to parse dataset info JSON." app.logger.debug(message) - return { 'success': False, - 'message': message, - 'job_id': None }, 500 + return {'success': False, + 'message': message, + 'job_id': None}, 500 # update try: return updateDataset(info) - except Exception, e: - message = "Failed index dataset. {0}:{1}\n{2}".format(type(e),e,traceback.format_exc()) + except Exception as e: + message = "Failed index dataset. {0}:{1}\n{2}".format( + type(e), e, traceback.format_exc()) app.logger.debug(message) - return { 'success': False, - 'message': message }, 500 + return {'success': False, + 'message': message}, 500 + + @hysds_io_ns.route('/list', endpoint='hysds_io-list') -@api.doc(responses={ 200: "Success", - 500: "Query execution failed" }, +@api.doc(responses={200: "Success", + 500: "Query execution failed"}, description="Gets list of registered hysds-io specifications and return as JSON.") class GetHySDSIOTypes(Resource): """Get list of registered hysds-io and return as JSON.""" @@ -89,19 +108,22 @@ def get(self): List HySDS IO specifications ''' try: - ids = hysds_commons.hysds_io_utils.get_hysds_io_types(app.config["ES_URL"],logger=app.logger) + ids = hysds_commons.hysds_io_utils.get_hysds_io_types( + app.config["ES_URL"], logger=app.logger) except Exception as e: - message = "Failed to query ES for HySDS IO types. {0}:{1}".format(type(e),str(e)) + message = "Failed to query ES for HySDS IO types. {0}:{1}".format( + type(e), str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 - return { 'success': True, - 'message': "", - 'result': ids } + return {'success': True, + 'message': "", + 'result': ids} + @hysds_io_ns.route('/type', endpoint='hysds_io-type') -@api.doc(responses={ 200: "Success", - 500: "Queue listing failed" }, +@api.doc(responses={200: "Success", + 500: "Queue listing failed"}, description="Gets info on a hysds-io specification.") class GetHySDSIOType(Resource): """Get list of job queues and return as JSON.""" @@ -113,7 +135,7 @@ class GetHySDSIOType(Resource): 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.Raw(required=True, description="HySDS IO Object") - }) + }) parser = api.parser() parser.add_argument('id', required=True, type=str, help="HySDS IO Type ID") @@ -124,20 +146,23 @@ def get(self): Gets a HySDS-IO specification by ID ''' try: - ident = request.form.get('id',request.args.get('id', None)) - spec = hysds_commons.hysds_io_utils.get_hysds_io(app.config["ES_URL"],ident,logger=app.logger) + ident = request.form.get('id', request.args.get('id', None)) + spec = hysds_commons.hysds_io_utils.get_hysds_io( + app.config["ES_URL"], ident, logger=app.logger) except Exception as e: - message = "Failed to query ES for HySDS IO object. {0}:{1}".format(type(e),str(e)) + message = "Failed to query ES for HySDS IO object. {0}:{1}".format( + type(e), str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 - return { 'success': True, - 'message': "", - 'result': spec } + return {'success': True, + 'message': "", + 'result': spec} + @hysds_io_ns.route('/add', endpoint='hysds_io-add') -@api.doc(responses={ 200: "Success", - 500: "Adding JSON failed" }, +@api.doc(responses={200: "Success", + 500: "Adding JSON failed"}, description="Adds a hysds-io specification") class AddHySDSIOType(Resource): """Add job spec""" @@ -149,9 +174,10 @@ class AddHySDSIOType(Resource): 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.String(required=True, description="HySDS IO ID") - }) + }) parser = api.parser() - parser.add_argument('spec', required=True, type=str, help="HySDS IO JSON Object") + parser.add_argument('spec', required=True, type=str, + help="HySDS IO JSON Object") @api.expect(parser) @api.marshal_with(resp_model) @@ -160,23 +186,26 @@ def post(self): Add a HySDS IO specification ''' try: - spec = request.form.get('spec',request.args.get('spec', None)) + spec = request.form.get('spec', request.args.get('spec', None)) if spec is None: raise Exception("'spec' must be supplied") obj = json.loads(spec) - ident = hysds_commons.hysds_io_utils.add_hysds_io(app.config["ES_URL"],obj,logger=app.logger) + ident = hysds_commons.hysds_io_utils.add_hysds_io( + app.config["ES_URL"], obj, logger=app.logger) except Exception as e: - message = "Failed to add ES for HySDS IO. {0}:{1}".format(type(e),str(e)) + message = "Failed to add ES for HySDS IO. {0}:{1}".format( + type(e), str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 - return { 'success': True, - 'message': "", - 'result': ident } + return {'success': True, + 'message': "", + 'result': ident} + @hysds_io_ns.route('/remove', endpoint='hysds_io-remove') -@api.doc(responses={ 200: "Success", - 500: "Remove JSON failed" }, +@api.doc(responses={200: "Success", + 500: "Remove JSON failed"}, description="Removes a hysds-io specification.") class RemoveHySDSIOType(Resource): """Remove job spec""" @@ -187,7 +216,7 @@ class RemoveHySDSIOType(Resource): "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), - }) + }) parser = api.parser() parser.add_argument('id', required=True, type=str, help="HySDS IO ID") @@ -198,13 +227,14 @@ def get(self): Remove HySDS IO for the given ID ''' try: - ident = request.form.get('id',request.args.get('id', None)) - hysds_commons.hysds_io_utils.remove_hysds_io(app.config["ES_URL"],ident,logger=app.logger) + ident = request.form.get('id', request.args.get('id', None)) + hysds_commons.hysds_io_utils.remove_hysds_io( + app.config["ES_URL"], ident, logger=app.logger) except Exception as e: - message = "Failed to add ES for HySDS IO. {0}:{1}".format(type(e),str(e)) + message = "Failed to add ES for HySDS IO. {0}:{1}".format( + type(e), str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 - return { 'success': True, - 'message': ""} - + return {'success': True, + 'message': ""} diff --git a/grq2/services/api_v02.py b/grq2/services/api_v02.py index de993ed..d1c1641 100644 --- a/grq2/services/api_v02.py +++ b/grq2/services/api_v02.py @@ -1,4 +1,15 @@ -import os, sys, json, requests, types, re +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import os +import sys +import json +import requests +import types +import re from flask import jsonify, Blueprint, request, Response, render_template, make_response from flask_restplus import Api, apidoc, Resource, fields diff --git a/grq2/services/geonames.py b/grq2/services/geonames.py index 65cd748..db0a99d 100644 --- a/grq2/services/geonames.py +++ b/grq2/services/geonames.py @@ -1,4 +1,14 @@ -import json, requests, types +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import int +from builtins import str +from future import standard_library +standard_library.install_aliases() +import json +import requests +import types from flask import jsonify, Blueprint, request, Response, render_template, make_response from pprint import pformat @@ -46,7 +56,7 @@ def cities(): # get results try: cities = get_cities(polygon, pop_th, size) - except Exception, e: + except Exception as e: return jsonify({ 'success': False, 'message': str(e), @@ -88,7 +98,7 @@ def continents(): # get results try: continents = get_continents(float(lon), float(lat)) - except Exception, e: + except Exception as e: return jsonify({ 'success': False, 'message': str(e), diff --git a/grq2/services/main.py b/grq2/services/main.py index db81d16..81f6879 100644 --- a/grq2/services/main.py +++ b/grq2/services/main.py @@ -1,9 +1,16 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from flask import jsonify, Blueprint from grq2 import app mod = Blueprint('services/main', __name__) + @mod.route('/services') def index(): return jsonify({'success': True, diff --git a/grq2/services/query.py b/grq2/services/query.py index aef8027..d452187 100644 --- a/grq2/services/query.py +++ b/grq2/services/query.py @@ -1,4 +1,14 @@ -import json, requests, types, re +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import map +from future import standard_library +standard_library.install_aliases() +import json +import requests +import types +import re from flask import jsonify, Blueprint, request, Response, render_template, make_response from pprint import pformat from shapely.geometry import box @@ -18,7 +28,7 @@ def datasets(): return jsonify({ 'success': True, 'message': "", - 'datasets': app.config['GRQ_DATASET_DOCTYPES'].keys() + 'datasets': list(app.config['GRQ_DATASET_DOCTYPES'].keys()) }) @@ -59,7 +69,7 @@ def grqByID(): # get dataset dataset = None - for ds, regex in app.config['OBJECTID_DATASET_MAP'].items(): + for ds, regex in list(app.config['OBJECTID_DATASET_MAP'].items()): match = re.search(regex, objectid) if match: dataset = ds @@ -78,10 +88,11 @@ def grqByID(): if version is not None: if version == 'latest': version = app.config['GRQ_UPDATE_INDEX_VERSION'][dataset] - index = '%s_%s_%s' % (app.config['GRQ_INDEX'], version.replace('.', ''), dataset.lower()) + index = '%s_%s_%s' % ( + app.config['GRQ_INDEX'], version.replace('.', ''), dataset.lower()) else: index = '%s_%s' % (app.config['GRQ_INDEX'], dataset.lower()) - + # query es_url = app.config['ES_URL'] query = { @@ -91,18 +102,20 @@ def grqByID(): ], "query": { "ids": { - "values": [ objectid ] + "values": [objectid] } } } #app.logger.debug("ES query for grqByID(): %s" % json.dumps(query, indent=2)) - r = requests.post('%s/%s/_search' % (es_url, index), data=json.dumps(query)) + r = requests.post('%s/%s/_search' % + (es_url, index), data=json.dumps(query)) res = r.json() result = [] if res['hits']['total'] > 0: # emulate result format from ElasticSearch <1.0 if '_source' in res['hits']['hits'][0]: - res['hits']['hits'][0]['fields'].update(res['hits']['hits'][0]['_source']) + res['hits']['hits'][0]['fields'].update( + res['hits']['hits'][0]['_source']) # get urls and metadata urls = res['hits']['hits'][0]['fields']['urls'] @@ -112,11 +125,13 @@ def grqByID(): metadata['id'] = res['hits']['hits'][0]['fields']['id'] # add data system version - metadata['data_system_version'] = res['hits']['hits'][0]['fields'].get('system_version', None) - + metadata['data_system_version'] = res['hits']['hits'][0]['fields'].get( + 'system_version', None) + # add dataset - metadata['dataset'] = res['hits']['hits'][0]['fields'].get('dataset', None) - + metadata['dataset'] = res['hits']['hits'][0]['fields'].get( + 'dataset', None) + # return plain list of urls if response_format == 'text': if response_groups not in ['Url', 'Urls']: @@ -131,11 +146,13 @@ def grqByID(): return response # return metadata with appropriate urls - if response_groups == 'Large': metadata['url'] = urls + if response_groups == 'Large': + metadata['url'] = urls else: - if len(urls) > 0: metadata['url'] = urls[0] + if len(urls) > 0: + metadata['url'] = urls[0] - result.append(metadata) + result.append(metadata) return jsonify({ 'success': success, @@ -217,8 +234,10 @@ def grq(): dataset = request.args.get('dataset', None) level = request.args.get('level', None) version = request.args.get('version', None) - starttime = request.args.get('starttime', request.args.get('sensingStart', None)) - endtime = request.args.get('endtime', request.args.get('sensingStop', None)) + starttime = request.args.get( + 'starttime', request.args.get('sensingStart', None)) + endtime = request.args.get( + 'endtime', request.args.get('sensingStop', None)) lat_min = request.args.get('lat_min', request.args.get('latMin', -90.)) lat_max = request.args.get('lat_max', request.args.get('latMax', 90.)) lon_min = request.args.get('lon_min', request.args.get('lonMin', -180.)) @@ -245,13 +264,14 @@ def grq(): # loop through for non-standard query params other_params = {} - for k in request.args.keys(): + for k in list(request.args.keys()): if k in ('dataset', 'level', 'version', 'starttime', 'endtime', 'lat_min', 'lat_max', 'lon_min', 'lon_max', 'responseGroups', 'format', 'sensingStart', 'sensingStop', 'latMin', 'latMax', - 'lonMin', 'lonMax', 'spatial'): continue + 'lonMin', 'lonMax', 'spatial'): + continue other_params[k] = request.args.get(k) - #app.logger.debug(pformat(other_params)) + # app.logger.debug(pformat(other_params)) # if no dataset passed, show help if dataset is None: @@ -290,19 +310,20 @@ def grq(): loc_filter_box = None else: loc_filter = { - "geo_shape" : { - "location" : { + "geo_shape": { + "location": { "shape": { "type": "envelope", "coordinates": [ - [ lon_min, lat_max ], - [ lon_max, lat_min ] + [lon_min, lat_max], + [lon_max, lat_min] ] } } } } - loc_filter_box = box(*map(float, [lon_min, lat_min, lon_max, lat_max])) + loc_filter_box = box( + *list(map(float, [lon_min, lat_min, lon_max, lat_max]))) # build query query = { @@ -316,44 +337,50 @@ def grq(): "_source" ], "query": { - "term": { "dataset": dataset } + "term": {"dataset": dataset} } } # add filters or query_string queries qs_queries = [] filters = [] - if date_filter is not None: filters.append(date_filter) - if loc_filter is not None: filters.append(loc_filter) + if date_filter is not None: + filters.append(date_filter) + if loc_filter is not None: + filters.append(loc_filter) for k in other_params: # fields to run query_string on (exact values and ranges) if k in ('latitudeIndexMin', 'latitudeIndexMax'): qs_query = { 'query_string': { - 'fields': [ k ], + 'fields': [k], 'query': other_params[k] } } qs_queries.append(qs_query) else: - term_filter = { 'term': {} } + term_filter = {'term': {}} term_filter['term']['metadata.%s' % k] = other_params[k] filters.append(term_filter) - if len(filters) > 0: query['filter'] = { "and": filters } - if len(qs_queries) > 0: query['query'] = { "bool": { "must": qs_queries } } + if len(filters) > 0: + query['filter'] = {"and": filters} + if len(qs_queries) > 0: + query['query'] = {"bool": {"must": qs_queries}} # get index if version is not None: if version == 'latest': version = app.config['GRQ_UPDATE_INDEX_VERSION'][es_index] - index = '%s_%s_%s' % (app.config['GRQ_INDEX'], version.replace('.', ''), es_index.lower()) + index = '%s_%s_%s' % ( + app.config['GRQ_INDEX'], version.replace('.', ''), es_index.lower()) else: index = '%s_%s' % (app.config['GRQ_INDEX'], es_index.lower()) - + # query for results es_url = app.config['ES_URL'] #app.logger.debug("ES query for grq(): %s" % json.dumps(query, indent=2)) - r = requests.post('%s/%s/_search?search_type=scan&scroll=10m&size=100' % (es_url, index), data=json.dumps(query)) + r = requests.post('%s/%s/_search?search_type=scan&scroll=10m&size=100' % + (es_url, index), data=json.dumps(query)) if r.status_code != 200: return jsonify({ 'success': False, @@ -366,20 +393,24 @@ def grq(): scroll_id = scan_result['_scroll_id'] results = [] while True: - r = requests.post('%s/_search/scroll?scroll=10m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=10m' % + es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: # emulate result format from ElasticSearch <1.0 - if '_source' in hit: hit['fields'].update(hit['_source']) + if '_source' in hit: + hit['fields'].update(hit['_source']) # spatial filter within vs. intersects (default) if spatial == 'within' and loc_filter_box is not None: - #print hit['fields']['location'] + # print hit['fields']['location'] lr = LinearRing(hit['fields']['location']['coordinates'][0]) - #print loc_filter_box, lr - if not lr.within(loc_filter_box): continue + # print loc_filter_box, lr + if not lr.within(loc_filter_box): + continue # get browse_urls, urls and metadata browse_urls = hit['fields']['browse_urls'] @@ -388,24 +419,27 @@ def grq(): # add objectid as id metadata['id'] = hit['fields']['id'] - + # add location if 'location' not in metadata: metadata['location'] = hit['fields']['location'] # add data system version - metadata['data_system_version'] = hit['fields'].get('system_version', None) - + metadata['data_system_version'] = hit['fields'].get( + 'system_version', None) + # add dataset metadata['dataset'] = hit['fields'].get('dataset', None) - + # return metadata with appropriate urls if response_groups == 'Large': metadata['browse_url'] = browse_urls metadata['url'] = urls else: - if len(urls) > 0: metadata['url'] = urls[0] - if len(browse_urls) > 0: metadata['browse_url'] = browse_urls[0] + if len(urls) > 0: + metadata['url'] = urls[0] + if len(browse_urls) > 0: + metadata['browse_url'] = browse_urls[0] results.append(metadata) @@ -413,9 +447,10 @@ def grq(): if response_format == 'text': urls = [] for m in results: - if isinstance(m['url'], types.ListType) and len(m['url']) > 0: + if isinstance(m['url'], list) and len(m['url']) > 0: urls.append(m['url'][0]) - else: urls.append(m['url']) + else: + urls.append(m['url']) if response_groups not in ['Url', 'Urls']: return jsonify({ 'success': False, @@ -450,8 +485,8 @@ def grq_es(): "query": { "match_all": {} }, - "sort":[{"_timestamp":{"order":"desc"}}], - "fields":["_timestamp","_source"] + "sort": [{"_timestamp": {"order": "desc"}}], + "fields": ["_timestamp", "_source"] } # get query @@ -466,7 +501,8 @@ def grq_es(): es_url = app.config['ES_URL'] index = '%s_%s' % (app.config['GRQ_INDEX'], dataset.lower()) #app.logger.debug("ES query for grq(): %s" % json.dumps(query, indent=2)) - r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, index), data=source) + r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, index), data=source) if r.status_code != 200: return jsonify({ 'success': False, @@ -483,34 +519,40 @@ def stream_json(scroll_id): yield '{\n "count": %d,\n "message": "",\n "result": [' % count res_count = 0 while True: - r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=60m' % + es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: res_count += 1 # emulate result format from ElasticSearch <1.0 - if '_source' in hit: hit['fields'].update(hit['_source']) + if '_source' in hit: + hit['fields'].update(hit['_source']) metadata = hit['fields']['metadata'] - + # get browse_urls, urls and metadata metadata['browse_url'] = hit['fields'].get('browse_urls', []) metadata['url'] = hit['fields'].get('urls', []) - + # add objectid as id metadata['id'] = hit['fields']['id'] - + # add location metadata['location'] = hit['fields'].get('location', None) - + # add data system version - metadata['data_system_version'] = hit['fields'].get('system_version', None) - + metadata['data_system_version'] = hit['fields'].get( + 'system_version', None) + # add dataset metadata['dataset'] = hit['fields'].get('dataset', None) - - if res_count == 1: yield '\n%s' % json.dumps(metadata, indent=2) - else: yield ',\n%s' % json.dumps(metadata, indent=2) + + if res_count == 1: + yield '\n%s' % json.dumps(metadata, indent=2) + else: + yield ',\n%s' % json.dumps(metadata, indent=2) yield '\n ],\n "success": true\n}' return Response(stream_json(scroll_id), mimetype="application/json") diff --git a/grq2/static/Leaflet.draw/package.json b/grq2/static/Leaflet.draw/package.json index d630fb7..c3d9a75 100644 --- a/grq2/static/Leaflet.draw/package.json +++ b/grq2/static/Leaflet.draw/package.json @@ -5,7 +5,7 @@ "devDependencies": { "leaflet": "git://github.com/Leaflet/Leaflet.git", "jshint": "~1.1.0", - "uglify-js": "~2.2.5", + "uglify-js": ">=2.6.0", "jake": "~0.5.10", "mocha": "~1.9.0", "karma": "~0.8.0" diff --git a/grq2/views/main.py b/grq2/views/main.py index 73d8ea5..1a6dcf5 100644 --- a/grq2/views/main.py +++ b/grq2/views/main.py @@ -1,3 +1,10 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from builtins import str +from future import standard_library +standard_library.install_aliases() from datetime import datetime from flask import render_template, Blueprint diff --git a/run.py b/run.py index 4e74120..e4ee947 100644 --- a/run.py +++ b/run.py @@ -1,4 +1,10 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from grq2 import app if __name__ == '__main__': - app.run(host="0.0.0.0", port=8878, debug=True) + app.run(host="0.0.0.0", port=8878, debug=True) diff --git a/run_debug.py b/run_debug.py index 611e9f2..28273de 100644 --- a/run_debug.py +++ b/run_debug.py @@ -1,4 +1,10 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() from grq2 import app if __name__ == '__main__': - app.run(host="0.0.0.0", port=8868, debug=True) + app.run(host="0.0.0.0", port=8868, debug=True) diff --git a/scripts/add_io_users.py b/scripts/add_io_users.py index 15a6f46..25fb3e0 100755 --- a/scripts/add_io_users.py +++ b/scripts/add_io_users.py @@ -1,6 +1,17 @@ #!/usr/bin/env python + from __future__ import print_function -import json, requests, sys, traceback +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import input +from builtins import str +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys +import traceback import re import fnmatch @@ -8,39 +19,44 @@ USER_RE = re.compile("^([a-zA-Z0-9-_.@]+\s*)+$") es_url = app.config['ES_URL'] + + def get_matching_hysdsios(): # get connection and create destination index # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - }, - "sort": [{"_id":{"order":"asc"}}] + "fields": "_source", + "query": { + "match_all": {} + }, + "sort": [{"_id": {"order": "asc"}}] } - r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, "hysds_ios"), data=json.dumps(query)) + r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, "hysds_ios"), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] users = {} print("Available HySDS IOs:") while True: - r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=60m' % + es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: ident = hit["_id"] - accounts = hit.get("_source",{}).get("allowed_accounts",[]) + accounts = hit.get("_source", {}).get("allowed_accounts", []) print("\t{0}: {1}".format(ident, json.dumps(accounts))) users[ident] = accounts - #Enter in valid HySDS - IO + # Enter in valid HySDS - IO print("-----------") hysdsios = [] while len(hysdsios) == 0: - pattern = raw_input("Enter in hysds-io id glob pattern (from above):") - hysdsios = fnmatch.filter(users.keys(),pattern) + pattern = eval(input("Enter in hysds-io id glob pattern (from above):")) + hysdsios = fnmatch.filter(list(users.keys()), pattern) if len(hysdsios) == 0: print("No matching hysds-ios for '{0}'".format(pattern)) print() @@ -48,20 +64,25 @@ def get_matching_hysdsios(): print("Selected hysds-io(s): {0}".format(json.dumps(hysdsios))) print("----------") return (hysdsios, users) + + def get_add_users(): - #User input + # User input user = "" while not USER_RE.match(user): if user != "": print("Invalid username(s): '{0}'".format(user)) - user = raw_input("Enter in space-separated user(s) to add:") + user = eval(input("Enter in space-separated user(s) to add:")) user = user.split() return user + + def add_to_ios(users, hysdsio, existing): ''' ''' if set(users).union(set(existing[hysdsio])) == set(existing[hysdsio]): - print("Supplied users already have permission to run {0}, nothing to do, Skipping.".format(hysdsio)) + print("Supplied users already have permission to run {0}, nothing to do, Skipping.".format( + hysdsio)) return existing[hysdsio] = list(set(existing[hysdsio]).union(set(users))) print() @@ -69,7 +90,8 @@ def add_to_ios(users, hysdsio, existing): print("------------") sure = "" while sure == "": - sure = raw_input("Are you sure you want to add {0} to allowed users for '{1}' for a final user set of {2}?" .format(users,hysdsio,json.dumps(existing[hysdsio]))) + sure = eval(input("Are you sure you want to add {0} to allowed users for '{1}' for a final user set of {2}?" .format( + users, hysdsio, json.dumps(existing[hysdsio])))) if not sure.startswith("y") and sure != "": print("User showed weakness, skipping") return @@ -77,14 +99,18 @@ def add_to_ios(users, hysdsio, existing): "doc": { "allowed_accounts": existing[hysdsio] }, - "doc_as_upsert":True + "doc_as_upsert": True } try: - r = requests.post('%s/%s/%s/%s/_update' % (es_url, "hysds_ios","hysds_io",hysdsio), data=json.dumps(doc)) + r = requests.post('%s/%s/%s/%s/_update' % (es_url, + "hysds_ios", "hysds_io", hysdsio), data=json.dumps(doc)) r.raise_for_status() except Exception as e: - print("[ERROR] Failed to update hysds-io. Resolve IMMEDIATELY. {0}:{1}\n{2}".format(type(e), str(e), traceback.format_exc()), file=sys.stderr) + print("[ERROR] Failed to update hysds-io. Resolve IMMEDIATELY. {0}:{1}\n{2}".format( + type(e), str(e), traceback.format_exc()), file=sys.stderr) sys.exit(-1) + + if __name__ == "__main__": hysdsios, existing = get_matching_hysdsios() users = get_add_users() diff --git a/scripts/clean_out_ios.py b/scripts/clean_out_ios.py index 01c4b38..4c7c60d 100755 --- a/scripts/clean_out_ios.py +++ b/scripts/clean_out_ios.py @@ -1,5 +1,14 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import input +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from grq2 import app @@ -9,13 +18,14 @@ # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - }, - "sort": [{"_id":{"order":"asc"}}] + "fields": "_source", + "query": { + "match_all": {} + }, + "sort": [{"_id": {"order": "asc"}}] } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, "hysds_ios"), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, "hysds_ios"), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -24,15 +34,16 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: ident = hit["_id"] - yn = raw_input("Do you want to delete: %s " % ident) + yn = eval(input("Do you want to delete: %s " % ident)) if yn == "y": - sure = raw_input("Are you sure? ID: %s " % ident) + sure = eval(input("Are you sure? ID: %s " % ident)) if not sure.startswith("y") and sure != "": - print "Skipping: %s" % ident + print(("Skipping: %s" % ident)) continue - r = requests.delete("%s/hysds_ios/hysds_io/%s" % (es_url,ident)) + r = requests.delete("%s/hysds_ios/hysds_io/%s" % (es_url, ident)) r.raise_for_status() - print "Deleted: %s" % ident + print(("Deleted: %s" % ident)) diff --git a/scripts/copy_add_groups_metadata.py b/scripts/copy_add_groups_metadata.py index 1b8beca..4bb79a2 100755 --- a/scripts/copy_add_groups_metadata.py +++ b/scripts/copy_add_groups_metadata.py @@ -1,5 +1,13 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from elasticsearch import Elasticsearch from grq2 import app @@ -17,12 +25,13 @@ # index all docs from source index to destination index query = { - "query": { - "match_all": {} - }, - "sort":[{"_timestamp":{"order":"desc"}}] + "query": { + "match_all": {} + }, + "sort": [{"_timestamp": {"order": "desc"}}] } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -31,10 +40,12 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] - if 'metadata' not in doc: continue + if 'metadata' not in doc: + continue doc['metadata']['groups'] = [ 'ancillary', 'CH4', @@ -46,5 +57,6 @@ 'radiation', 'temperature', ] - ret = es.index(index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) - print "indexed %s" % hit['_id'] + ret = es.index( + index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) + print(("indexed %s" % hit['_id'])) diff --git a/scripts/copy_add_new_metadata.py b/scripts/copy_add_new_metadata.py index 9c8f01d..f19aba9 100755 --- a/scripts/copy_add_new_metadata.py +++ b/scripts/copy_add_new_metadata.py @@ -1,5 +1,13 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from elasticsearch import Elasticsearch from grq2 import app @@ -17,13 +25,14 @@ # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - }, - "sort":[{"_timestamp":{"order":"desc"}}] + "fields": "_source", + "query": { + "match_all": {} + }, + "sort": [{"_timestamp": {"order": "desc"}}] } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -32,10 +41,12 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] - if 'metadata' not in doc: continue + if 'metadata' not in doc: + continue doc['metadata']['resolution'] = [ "(1 day, 40 standard pressure levels, .07 degree, .07 degree)", ] @@ -43,5 +54,6 @@ "(time, level, lat, lon)", ] doc['metadata']['dimensions'] = 4 - ret = es.index(index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) - print "indexed %s" % hit['_id'] + ret = es.index( + index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) + print(("indexed %s" % hit['_id'])) diff --git a/scripts/copy_add_resolutions_airx3ret.py b/scripts/copy_add_resolutions_airx3ret.py index 7e71474..ae7f272 100755 --- a/scripts/copy_add_resolutions_airx3ret.py +++ b/scripts/copy_add_resolutions_airx3ret.py @@ -1,5 +1,13 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from elasticsearch import Elasticsearch from grq2 import app @@ -17,13 +25,14 @@ # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - }, - "sort":[{"_timestamp":{"order":"desc"}}] + "fields": "_source", + "query": { + "match_all": {} + }, + "sort": [{"_timestamp": {"order": "desc"}}] } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -32,10 +41,12 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] - if 'metadata' not in doc: continue + if 'metadata' not in doc: + continue doc['dataset'] = "AIRX3RET-daily-subset" doc['metadata']['data_product_name'] = "AIRX3RET-daily-subset" doc['metadata']['temporal_resolution'] = "1 day" @@ -43,5 +54,6 @@ "(24 standard pressure levels, 1 degree, 1 degree)", "(12 H2O pressure layers, 1 degree, 1 degree)", ] - ret = es.index(index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) - print "indexed %s" % hit['_id'] + ret = es.index( + index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) + print(("indexed %s" % hit['_id'])) diff --git a/scripts/copy_add_resolutions_airx3spd.py b/scripts/copy_add_resolutions_airx3spd.py index f27d85b..b384c55 100755 --- a/scripts/copy_add_resolutions_airx3spd.py +++ b/scripts/copy_add_resolutions_airx3spd.py @@ -1,5 +1,13 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from elasticsearch import Elasticsearch from grq2 import app @@ -17,13 +25,14 @@ # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - }, - "sort":[{"_timestamp":{"order":"desc"}}] + "fields": "_source", + "query": { + "match_all": {} + }, + "sort": [{"_timestamp": {"order": "desc"}}] } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -32,10 +41,12 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] - if 'metadata' not in doc: continue + if 'metadata' not in doc: + continue doc['dataset'] = "AIRX3SPD-daily-subset" doc['metadata']['data_product_name'] = "AIRX3SPD-daily-subset" doc['metadata']['temporal_resolution'] = "1 day" @@ -43,5 +54,6 @@ "(100 extra pressure levels, 1 degree, 1 degree)", "(7 cloud phases, 1 degree, 1 degree)", ] - ret = es.index(index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) - print "indexed %s" % hit['_id'] + ret = es.index( + index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) + print(("indexed %s" % hit['_id'])) diff --git a/scripts/copy_index.py b/scripts/copy_index.py index 7f4e4b7..28a75d6 100755 --- a/scripts/copy_index.py +++ b/scripts/copy_index.py @@ -1,5 +1,13 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from elasticsearch import Elasticsearch from grq2 import app @@ -17,12 +25,13 @@ # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - } + "fields": "_source", + "query": { + "match_all": {} + } } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -31,8 +40,10 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] - ret = es.index(index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) - print "indexed %s" % hit['_id'] + ret = es.index( + index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) + print(("indexed %s" % hit['_id'])) diff --git a/scripts/copy_modify_index.py b/scripts/copy_modify_index.py index 96345ff..054d412 100755 --- a/scripts/copy_modify_index.py +++ b/scripts/copy_modify_index.py @@ -1,5 +1,13 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from elasticsearch import Elasticsearch from grq2 import app @@ -17,11 +25,12 @@ # index all docs from source index to destination index query = { - "query": { - "match_all": {} - } + "query": { + "match_all": {} + } } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -30,12 +39,14 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] - #if 'metadata' not in doc: continue + # if 'metadata' not in doc: continue if 'version' in doc: doc['system_version'] = doc['version'] del doc['version'] - ret = es.index(index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) - print "indexed %s" % hit['_id'] + ret = es.index( + index=dest, doc_type=hit['_type'], id=hit['_id'], body=doc) + print(("indexed %s" % hit['_id'])) diff --git a/scripts/copy_user_rules.py b/scripts/copy_user_rules.py index 29fa983..3e4f488 100755 --- a/scripts/copy_user_rules.py +++ b/scripts/copy_user_rules.py @@ -1,5 +1,14 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import open +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from grq2 import app from grq2.lib.utils import parse_config @@ -29,12 +38,13 @@ # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - } + "fields": "_source", + "query": { + "match_all": {} + } } -r = requests.post('%s/%s/.percolator/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/.percolator/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -43,16 +53,18 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] #doc['query_string'] = doc['query_string'].replace('spacecraftName', 'platform') #doc['query'] = json.loads(doc['query_string']) #conn.index(hit['_source'], dest, '.percolator', hit['_id']) - r = requests.post("%s/%s/.percolator/" % (es_url, dest), data=json.dumps(doc)) + r = requests.post("%s/%s/.percolator/" % + (es_url, dest), data=json.dumps(doc)) result = r.json() if r.status_code != 201: - print "Failed to insert rule: %s" % json.dumps(doc, indent=2) + print(("Failed to insert rule: %s" % json.dumps(doc, indent=2))) continue - - print "indexed %s" % hit['_id'] + + print(("indexed %s" % hit['_id'])) diff --git a/scripts/copy_user_tags.py b/scripts/copy_user_tags.py index dc1d3fc..08a7a36 100755 --- a/scripts/copy_user_tags.py +++ b/scripts/copy_user_tags.py @@ -1,5 +1,13 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from pprint import pprint from grq2 import app @@ -16,11 +24,12 @@ # index all docs from source index to destination index query = { - "query": { - "match_all": {} - } + "query": { + "match_all": {} + } } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -29,20 +38,23 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] user_tags = doc['metadata'].get('user_tags', None) - if user_tags is None: continue + if user_tags is None: + continue # upsert new document new_doc = { - "doc": { "metadata": { "user_tags": user_tags } }, + "doc": {"metadata": {"user_tags": user_tags}}, "doc_as_upsert": True } - r = requests.post('%s/%s/%s/%s/_update' % (es_url, dest, doc_type, hit['_id']), data=json.dumps(new_doc)) + r = requests.post('%s/%s/%s/%s/_update' % (es_url, dest, + doc_type, hit['_id']), data=json.dumps(new_doc)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to update user_tags for %s. Got status code %d:\n%s" % + app.logger.debug("Failed to update user_tags for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() diff --git a/scripts/create_index.py b/scripts/create_index.py index 976afed..30a03ee 100755 --- a/scripts/create_index.py +++ b/scripts/create_index.py @@ -1,5 +1,13 @@ #!/usr/bin/env python -import json, requests, sys +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys from elasticsearch import Elasticsearch from grq2 import app diff --git a/scripts/delete_s3_objects.py b/scripts/delete_s3_objects.py index 14b19e3..6b305e8 100755 --- a/scripts/delete_s3_objects.py +++ b/scripts/delete_s3_objects.py @@ -1,7 +1,14 @@ #!/usr/bin/env python +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import open +from future import standard_library +standard_library.install_aliases() import boto3 import sys -from urlparse import urlparse +from urllib.parse import urlparse file = open(sys.argv[1]) client = boto3.client('s3') @@ -9,7 +16,7 @@ for line in file: parsed_url = urlparse(line.strip()) bucket = parsed_url.hostname.split('.', 1)[0] - print 'Deleting bucket=%s, key=%s' % (bucket, parsed_url.path[1:]) + print(('Deleting bucket=%s, key=%s' % (bucket, parsed_url.path[1:]))) respone = client.delete_object(Bucket=bucket, Key=parsed_url.path[1:]) file.close() diff --git a/scripts/fix_temporal_span.py b/scripts/fix_temporal_span.py index bd12da0..8cc7b02 100755 --- a/scripts/fix_temporal_span.py +++ b/scripts/fix_temporal_span.py @@ -1,15 +1,26 @@ #!/usr/bin/env python -import sys, re, json, requests +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import map +from future import standard_library +standard_library.install_aliases() +import sys +import re +import json +import requests from datetime import datetime from grq2 import app -SENSING_RE = re.compile(r'(S1-.*?_(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2}).*?orb)') +SENSING_RE = re.compile( + r'(S1-.*?_(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2}).*?orb)') def getDatetimeFromString(dtStr, dayOnly=False): - (year,month,day,hour,minute,second) = getTimeElementsFromString(dtStr) + (year, month, day, hour, minute, second) = getTimeElementsFromString(dtStr) if dayOnly: return datetime(year=year, month=month, day=day) else: @@ -19,8 +30,8 @@ def getDatetimeFromString(dtStr, dayOnly=False): def getTemporalSpanInDays(dt1, dt2): temporal_diff = getDatetimeFromString(dt1) - getDatetimeFromString(dt2) - #print(temporal_diff.days) - #print(temporal_diff.seconds) + # print(temporal_diff.days) + # print(temporal_diff.seconds) temporal_span = abs(temporal_diff.days) if abs(temporal_diff.seconds) >= 43200.: temporal_span += 1 @@ -28,84 +39,92 @@ def getTemporalSpanInDays(dt1, dt2): def getTimeElementsFromString(dtStr): - match = re.match(r'^(\d{4})[/-](\d{2})[/-](\d{2})[\s*T](\d{2}):(\d{2}):(\d{2})(?:\.\d+)?Z?$',dtStr) - if match: (year,month,day,hour,minute,second) = map(int,match.groups()) + match = re.match( + r'^(\d{4})[/-](\d{2})[/-](\d{2})[\s*T](\d{2}):(\d{2}):(\d{2})(?:\.\d+)?Z?$', dtStr) + if match: + (year, month, day, hour, minute, second) = list(map(int, match.groups())) else: - match = re.match(r'^(\d{4})[/-](\d{2})[/-](\d{2})$',dtStr) + match = re.match(r'^(\d{4})[/-](\d{2})[/-](\d{2})$', dtStr) if match: - (year,month,day) = map(int,match.groups()) - (hour,minute,second) = (0,0,0) - else: raise(RuntimeError("Failed to recognize date format: %s" % dtStr)) - return (year,month,day,hour,minute,second) + (year, month, day) = list(map(int, match.groups())) + (hour, minute, second) = (0, 0, 0) + else: + raise RuntimeError + return (year, month, day, hour, minute, second) def main(): src = sys.argv[1] - + #id = "S1-IFG_RM_M1S1_TN120_20161101T232831-20161008T232803_s1-poeorb-7b8c-v1.1.2-standard" - + #match = SENSING_RE.search(id) - #sensing_start, sensing_stop = sorted(["%s-%s-%sT%s:%s:%s" % match.groups()[1:7], + # sensing_start, sensing_stop = sorted(["%s-%s-%sT%s:%s:%s" % match.groups()[1:7], # "%s-%s-%sT%s:%s:%s" % match.groups()[7:]]) #print(sensing_start, sensing_stop) - + # index all docs from source index to destination index query = { - "fields": "temporal_span", - "query": { - "match_all": {} - }, - "sort": [ - { - "creation_timestamp": { - "order": "asc" - } - } - ] + "fields": "temporal_span", + "query": { + "match_all": {} + }, + "sort": [ + { + "creation_timestamp": { + "order": "asc" + } + } + ] } - #query = { + # query = { # "fields": "_source", # "query": { # "match_all": {} # } - #} + # } es_url = app.config['ES_URL'] - r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) + r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] results = [] while True: - r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=60m' % + es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: - print(json.dumps(hit, indent=2)) + print((json.dumps(hit, indent=2))) id = hit['_id'] old_span = hit['fields']['temporal_span'][0] - print("old temporal span: %d" % old_span) + print(("old temporal span: %d" % old_span)) match = SENSING_RE.search(hit['_id']) sensing_start, sensing_stop = sorted(["%s-%s-%sT%s:%s:%s" % match.groups()[1:7], "%s-%s-%sT%s:%s:%s" % match.groups()[7:]]) new_span = getTemporalSpanInDays(sensing_stop, sensing_start) - print("new temporal span: %d" % new_span) + print(("new temporal span: %d" % new_span)) if new_span == old_span: - print("%s already fixed." % hit['_id']) + print(("%s already fixed." % hit['_id'])) continue # upsert new document new_doc = { - "doc": { "temporal_span": new_span }, + "doc": {"temporal_span": new_span}, "doc_as_upsert": True } - r = requests.post('%s/%s/%s/%s/_update' % (es_url, src, hit['_type'], hit['_id']), data=json.dumps(new_doc)) + r = requests.post('%s/%s/%s/%s/_update' % (es_url, src, + hit['_type'], hit['_id']), data=json.dumps(new_doc)) result = r.json() if r.status_code != 200: - app.logger.debug("Failed to update user_tags for %s. Got status code %d:\n%s" % + app.logger.debug("Failed to update user_tags for %s. Got status code %d:\n%s" % (hit['_id'], r.status_code, json.dumps(result, indent=2))) r.raise_for_status() -if __name__ == "__main__": main() +if __name__ == "__main__": + main() diff --git a/scripts/geonames/import_ES.py b/scripts/geonames/import_ES.py index ce61fd1..f6061dc 100755 --- a/scripts/geonames/import_ES.py +++ b/scripts/geonames/import_ES.py @@ -1,5 +1,17 @@ #!/usr/bin/env python -import string, traceback +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import open +from builtins import dict +from builtins import zip +from builtins import int +from builtins import map +from future import standard_library +standard_library.install_aliases() +import string +import traceback from elasticsearch import Elasticsearch from pprint import pprint @@ -64,74 +76,74 @@ DOCTYPE = 'geoname' MAPPING = { 'geoname': { - '_id': { 'type': 'string', 'path': 'geonameid', 'index': 'not_analyzed', 'store': 'yes' }, - '_timestamp': { 'enabled': True, 'store': 'yes' }, + '_id': {'type': 'string', 'path': 'geonameid', 'index': 'not_analyzed', 'store': 'yes'}, + '_timestamp': {'enabled': True, 'store': 'yes'}, 'properties': { - 'geonameid': { 'type': 'string', 'index': 'not_analyzed' }, + 'geonameid': {'type': 'string', 'index': 'not_analyzed'}, 'name': { 'type': 'multi_field', 'fields': { - 'name': { 'type': 'string', 'index': 'analyzed' }, - 'untouched': { 'type': 'string', 'index': 'not_analyzed' } + 'name': {'type': 'string', 'index': 'analyzed'}, + 'untouched': {'type': 'string', 'index': 'not_analyzed'} } }, 'asciiname': { 'type': 'multi_field', 'fields': { - 'asciiname': { 'type': 'string', 'index': 'analyzed' }, - 'untouched': { 'type': 'string', 'index': 'not_analyzed' } + 'asciiname': {'type': 'string', 'index': 'analyzed'}, + 'untouched': {'type': 'string', 'index': 'not_analyzed'} } }, 'alternatename': { 'type': 'multi_field', 'fields': { - 'alternatename': { 'type': 'string', 'index': 'analyzed' }, - 'untouched': { 'type': 'string', 'index': 'not_analyzed' } + 'alternatename': {'type': 'string', 'index': 'analyzed'}, + 'untouched': {'type': 'string', 'index': 'not_analyzed'} } }, - 'latitude': { 'type': 'double' }, - 'longitude': { 'type': 'double' }, - 'feature_class': { 'type': 'string', 'index': 'not_analyzed' }, - 'feature_code': { 'type': 'string', 'index': 'not_analyzed' }, - 'country_code': { 'type': 'string', 'index': 'not_analyzed' }, - 'cc2': { 'type': 'string', 'index': 'not_analyzed' }, - 'admin1_code': { 'type': 'string', 'index': 'not_analyzed' }, - 'admin2_code': { 'type': 'string', 'index': 'not_analyzed' }, - 'admin3_code': { 'type': 'string', 'index': 'not_analyzed' }, - 'admin4_code': { 'type': 'string', 'index': 'not_analyzed' }, - 'population': { 'type': 'long' }, - 'elevation': { 'type': 'long' }, - 'dem': { 'type': 'string' }, - 'timezone': { 'type': 'string' }, - 'modification_date': { 'type': 'date', 'format': 'dateOptionalTime' }, - 'location': { 'type': 'geo_point' }, - 'continent_code': { 'type': 'string', 'index': 'not_analyzed' }, + 'latitude': {'type': 'double'}, + 'longitude': {'type': 'double'}, + 'feature_class': {'type': 'string', 'index': 'not_analyzed'}, + 'feature_code': {'type': 'string', 'index': 'not_analyzed'}, + 'country_code': {'type': 'string', 'index': 'not_analyzed'}, + 'cc2': {'type': 'string', 'index': 'not_analyzed'}, + 'admin1_code': {'type': 'string', 'index': 'not_analyzed'}, + 'admin2_code': {'type': 'string', 'index': 'not_analyzed'}, + 'admin3_code': {'type': 'string', 'index': 'not_analyzed'}, + 'admin4_code': {'type': 'string', 'index': 'not_analyzed'}, + 'population': {'type': 'long'}, + 'elevation': {'type': 'long'}, + 'dem': {'type': 'string'}, + 'timezone': {'type': 'string'}, + 'modification_date': {'type': 'date', 'format': 'dateOptionalTime'}, + 'location': {'type': 'geo_point'}, + 'continent_code': {'type': 'string', 'index': 'not_analyzed'}, 'continent_name': { 'type': 'multi_field', 'fields': { - 'continent_name': { 'type': 'string', 'index': 'analyzed' }, - 'untouched': { 'type': 'string', 'index': 'not_analyzed' } + 'continent_name': {'type': 'string', 'index': 'analyzed'}, + 'untouched': {'type': 'string', 'index': 'not_analyzed'} } }, 'country_name': { 'type': 'multi_field', 'fields': { - 'country_name': { 'type': 'string', 'index': 'analyzed' }, - 'untouched': { 'type': 'string', 'index': 'not_analyzed' } + 'country_name': {'type': 'string', 'index': 'analyzed'}, + 'untouched': {'type': 'string', 'index': 'not_analyzed'} } }, 'admin1_name': { 'type': 'multi_field', 'fields': { - 'admin1_name': { 'type': 'string', 'index': 'analyzed' }, - 'untouched': { 'type': 'string', 'index': 'not_analyzed' } + 'admin1_name': {'type': 'string', 'index': 'analyzed'}, + 'untouched': {'type': 'string', 'index': 'not_analyzed'} } }, 'admin2_name': { 'type': 'multi_field', 'fields': { - 'admin2_name': { 'type': 'string', 'index': 'analyzed' }, - 'untouched': { 'type': 'string', 'index': 'not_analyzed' } + 'admin2_name': {'type': 'string', 'index': 'analyzed'}, + 'untouched': {'type': 'string', 'index': 'not_analyzed'} } } } @@ -149,17 +161,18 @@ def get_admin_dict(admin_file): vals = line.strip().split('\t') d[vals[0]] = vals[1:] return d - + def get_country_dict(country_file): d = {} with open(country_file) as f: for line in f: - if line.startswith('#'): continue + if line.startswith('#'): + continue vals = line.strip().split('\t') d[vals[0]] = vals[1:] return d - + def parse(csv_file): @@ -179,19 +192,22 @@ def parse(csv_file): try: with open(csv_file) as f: for line in f: - row = dict(zip(FIELD_NAMES, line.strip().split('\t'))) + row = dict(list(zip(FIELD_NAMES, line.strip().split('\t')))) line_number += 1 for k in row: if row[k] == '': - if k in ('alternatename', 'cc2'): row[k] = [] - else: row[k] = None + if k in ('alternatename', 'cc2'): + row[k] = [] + else: + row[k] = None else: if k in ('alternatename', 'cc2'): - row[k] = map(string.strip, row[k].split(',')) + row[k] = list(map(string.strip, row[k].split(','))) if k in ('latitude', 'longitude'): row[k] = float(row[k]) if k in ('population', 'elevation'): - if row[k] is not None: row[k] = int(row[k]) + if row[k] is not None: + row[k] = int(row[k]) # add geo_point location for spatial query row['location'] = { @@ -211,17 +227,20 @@ def parse(csv_file): row['continent_code'] = cntries[row['country_code']][7] row['continent_name'] = CONTINENT_CODES[row['continent_code']] if row['admin1_code'] is not None: - adm1_code = '%s.%s' % (row['country_code'], row['admin1_code']) + adm1_code = '%s.%s' % ( + row['country_code'], row['admin1_code']) row['admin1_name'] = adm1.get(adm1_code, [None])[0] if row['admin2_code'] is not None: - adm2_code = '%s.%s' % (adm1_code, row['admin2_code']) + adm2_code = '%s.%s' % ( + adm1_code, row['admin2_code']) row['admin2_name'] = adm2.get(adm2_code, [None])[0] - # index - ret = es.index(index=INDEX, doc_type=DOCTYPE, id=row['geonameid'], body=row) - except Exception, e: + # index + ret = es.index(index=INDEX, doc_type=DOCTYPE, + id=row['geonameid'], body=row) + except Exception as e: traceback.print_exc() - print "line_number: %d" % line_number + print(("line_number: %d" % line_number)) raise diff --git a/scripts/install_es_template.py b/scripts/install_es_template.py index 2a20a2d..0275385 100755 --- a/scripts/install_es_template.py +++ b/scripts/install_es_template.py @@ -1,5 +1,15 @@ #!/usr/bin/env python -import os, sys, json, requests +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import open +from future import standard_library +standard_library.install_aliases() +import os +import sys +import json +import requests from jinja2 import Template from grq2 import app @@ -14,8 +24,8 @@ def write_template(es_url, prefix, alias, tmpl_file): r = requests.delete(tmpl_url) r = requests.put(tmpl_url, data=tmpl) r.raise_for_status() - print r.json() - print "Successfully installed template %s at %s." % (alias, tmpl_url) + print((r.json())) + print(("Successfully installed template %s at %s." % (alias, tmpl_url))) if __name__ == "__main__": diff --git a/scripts/migrate_s3_datasets.py b/scripts/migrate_s3_datasets.py index df1e064..245591f 100755 --- a/scripts/migrate_s3_datasets.py +++ b/scripts/migrate_s3_datasets.py @@ -1,53 +1,67 @@ #!/usr/bin/env python -import json, requests +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests import sys + def migrate(index_name): src = index_name dest = index_name[0:index_name.index('_update')] - #index sizes should be the same on both indices before copying. If not, error out. + # index sizes should be the same on both indices before copying. If not, error out. r = requests.get('http://localhost:9200/%s/_count' % src) src_count = r.json()['count'] - + r = requests.get('http://localhost:9200/%s/_count' % dest) - dest_count = r.json()['count'] - + dest_count = r.json()['count'] + if src_count != dest_count: - print 'ERROR: index %s (%d) does not appear to match index %s (%d)' % (src, src_count, dest, dest_count) + print(('ERROR: index %s (%d) does not appear to match index %s (%d)' % + (src, src_count, dest, dest_count))) sys.exit(1) - + # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - } + "fields": "_source", + "query": { + "match_all": {} + } } - r = requests.post('http://localhost:9200/%s/_search?search_type=scan&scroll=60m&size=100' % src, data=json.dumps(query)) + r = requests.post( + 'http://localhost:9200/%s/_search?search_type=scan&scroll=60m&size=100' % src, data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] results = [] while True: - r = requests.post('http://localhost:9200/_search/scroll?scroll=60m', data=scroll_id) + r = requests.post( + 'http://localhost:9200/_search/scroll?scroll=60m', data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['_source'] # conn.index(hit['_source'], dest, hit['_type'], hit['_id']) - post_request = 'http://localhost:9200/%s/%s/%s' % (dest, hit['_type'], hit['_id']) + post_request = 'http://localhost:9200/%s/%s/%s' % ( + dest, hit['_type'], hit['_id']) r = requests.post(post_request, data=json.dumps(hit['_source'])) if r.status_code == 200: - print "INFO: indexed \"%s\" on \"%s\"" % (hit['_id'], dest) + print(("INFO: indexed \"%s\" on \"%s\"" % (hit['_id'], dest))) else: - print 'ERROR: post request returned status code %d: %s' % (r.status_code, post_request) - print(r.json()) + print(('ERROR: post request returned status code %d: %s' % + (r.status_code, post_request))) + print((r.json())) sys.exit(1) -# Query for all indices that end with _update + +# Query for all indices that end with _update r = requests.get('http://localhost:9200/*_update/_stats') result = r.json() -for index in result['indices'].keys(): +for index in list(result['indices'].keys()): migrate(index) - diff --git a/scripts/rename_s3_datasets.py b/scripts/rename_s3_datasets.py index cb4e907..d0c6d45 100755 --- a/scripts/rename_s3_datasets.py +++ b/scripts/rename_s3_datasets.py @@ -1,8 +1,16 @@ #!/usr/bin/env python -import json, requests +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import open +from future import standard_library +standard_library.install_aliases() +import json +import requests import boto3 -from urlparse import urlparse +from urllib.parse import urlparse import sys from elasticsearch import Elasticsearch @@ -13,43 +21,52 @@ file = open(sys.argv[1], 'w') client = boto3.client('s3') + def move_s3_files(url, target_path): parsed_url = urlparse(url) bucket = parsed_url.hostname.split('.', 1)[0] - results = client.list_objects(Bucket=bucket, Delimiter='/', Prefix=parsed_url.path[1:] + '/') - + results = client.list_objects( + Bucket=bucket, Delimiter='/', Prefix=parsed_url.path[1:] + '/') + if results.get('Contents'): for result in results.get('Contents'): - file_url = parsed_url.scheme + "://" + parsed_url.hostname + '/' + result.get('Key') + file_url = parsed_url.scheme + "://" + \ + parsed_url.hostname + '/' + result.get('Key') filename = result.get('Key').split('/')[-1] - print 'INFO: Copying object \"%s\" to \"%s\"' % (file_url, target_path + '/' + filename) + print(('INFO: Copying object \"%s\" to \"%s\"' % + (file_url, target_path + '/' + filename))) copy_source = bucket + '/' + result.get('Key') key = target_path + '/' + filename - r = client.copy_object(Bucket=bucket, CopySource=copy_source, Key=key) + r = client.copy_object( + Bucket=bucket, CopySource=copy_source, Key=key) if r['ResponseMetadata']['HTTPStatusCode'] != 200: - print 'ERROR: %d' % r['ResponseMetadata']['HTTPStatusCode'] - print 'ERROR: Problem occured copying object \"%s\"' % file_url - print 'ERROR: CopySource=%s, Key=%s' % (copy_source, key) + print(('ERROR: %d' % r['ResponseMetadata']['HTTPStatusCode'])) + print(('ERROR: Problem occured copying object \"%s\"' % file_url)) + print(('ERROR: CopySource=%s, Key=%s' % (copy_source, key))) sys.exit(1) else: - if r.has_key('Error'): - print 'ERROR: Problem occured copying object \"%s\"' % file_url - print 'ERROR: CopySource=%s, Key=%s' % (copy_source, key) - print 'ERROR: %s: %s' % (r['Error']['Code'], r['Error']['Message']) - sys.exit(1) + if 'Error' in r: + print(('ERROR: Problem occured copying object \"%s\"' % + file_url)) + print(('ERROR: CopySource=%s, Key=%s' % (copy_source, key))) + print(('ERROR: %s: %s' % + (r['Error']['Code'], r['Error']['Message']))) + sys.exit(1) else: # Delete object here? file.write(file_url + "\n") - + if results.get('CommonPrefixes'): for result in results.get('CommonPrefixes'): # Prefix values have a trailing '/'. Let's remove it to be consistent with our dir urls - folder = parsed_url.scheme + "://" + parsed_url.hostname + '/' + result.get('Prefix')[:-1] + folder = parsed_url.scheme + "://" + \ + parsed_url.hostname + '/' + result.get('Prefix')[:-1] # Get the sub-directory name sub_directory = result.get('Prefix')[:-1].split('/')[-1] updated_target_path = target_path + '/' + sub_directory move_s3_files(folder, updated_target_path) - + + def create_index(index, doctype): # get connection and create index @@ -61,20 +78,23 @@ def create_index(index, doctype): url_keys = ['urls', 'browse_urls'] # query -r = requests.get('http://localhost:9200/grq_aria/_search?search_type=scan&scroll=10m&size=100') +r = requests.get( + 'http://localhost:9200/grq_aria/_search?search_type=scan&scroll=10m&size=100') if r.status_code != 200: - app.logger.debug("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(r.json(), indent=2))) + app.logger.debug("Failed to query ES. Got status code %d:\n%s" % + (r.status_code, json.dumps(r.json(), indent=2))) r.raise_for_status() #app.logger.debug("result: %s" % pformat(r.json())) scan_result = r.json() scroll_id = scan_result['_scroll_id'] while (True): - r = requests.post('http://localhost:9200/_search/scroll?scroll=10m', data=scroll_id) + r = requests.post( + 'http://localhost:9200/_search/scroll?scroll=10m', data=scroll_id) datasets = r.json() scroll_id = datasets['_scroll_id'] if len(datasets['hits']['hits']) == 0: - file.close() + file.close() break for hit in datasets['hits']['hits']: updated_urls = {} @@ -87,26 +107,30 @@ def create_index(index, doctype): else: parsed_url = urlparse(url) if (parsed_url.scheme == 's3') or ('hysds-aria-products.s3' in parsed_url.hostname): - date = updated_doc['_source']['starttime'].split('T')[0].split('-') + date = updated_doc['_source']['starttime'].split('T')[ + 0].split('-') path = (updated_doc['_source']['dataset_type'] + '/' + updated_doc['_source']['system_version'] - + '/' + date[0] + '/' + date[1] + '/' + date[2] + '/' + + '/' + date[0] + '/' + + date[1] + '/' + date[2] + '/' + updated_doc['_source']['id']) if parsed_url.scheme == 's3': - # Parses out the date field from the timestamp and tokenizes it into year, month, day - new_urls.append(parsed_url.scheme + "://" + parsed_url.hostname + '/' + path) + # Parses out the date field from the timestamp and tokenizes it into year, month, day + new_urls.append( + parsed_url.scheme + "://" + parsed_url.hostname + '/' + path) elif 'hysds-aria-products.s3' in parsed_url.hostname: new_location = parsed_url.scheme + "://" + parsed_url.hostname + '/' + path if url != new_location: - print 'INFO: New target path: %s' % path + print(('INFO: New target path: %s' % path)) move_s3_files(url, path) new_urls.append(new_location) else: - print 'INFO: URL appears to already conform to correct naming convention: %s. Will not move.' % url + print(( + 'INFO: URL appears to already conform to correct naming convention: %s. Will not move.' % url)) new_urls.append(url) if new_urls: updated_urls[url_key] = new_urls - + if updated_urls: # sort the urls first before updating the document # Add s3-browse link to the end of the list @@ -121,17 +145,22 @@ def create_index(index, doctype): if s3_browse_url is not None: sorted_urls.append(s3_browse_url) updated_doc['_source'][url_key] = sorted_urls - + updated_index = updated_doc['_index'] + '_update' create_index(updated_index, updated_doc['_type']) - post_request = 'http://localhost:9200/%s/%s/%s' % (updated_index, updated_doc['_type'], updated_doc['_id']) - r = requests.post(post_request, data=json.dumps(updated_doc['_source'])) + post_request = 'http://localhost:9200/%s/%s/%s' % ( + updated_index, updated_doc['_type'], updated_doc['_id']) + r = requests.post( + post_request, data=json.dumps(updated_doc['_source'])) if r.status_code == 200 or r.status_code == 201: - print 'SUCCESS: Successfully posted updated document for %s in index %s' % (updated_doc['_id'], updated_index) + print(('SUCCESS: Successfully posted updated document for %s in index %s' % ( + updated_doc['_id'], updated_index))) r.raise_for_status() else: - print 'ERROR: Post request returned status code %d: %s' % (r.status_code, post_request) - print(r.json()) + print(('ERROR: Post request returned status code %d: %s' % + (r.status_code, post_request))) + print((r.json())) sys.exit(1) else: - print 'SKIP: Skipping \"%s\" dataset from index \"%s\". No S3 URLs to update' % (updated_doc['_id'], updated_doc['_index']) + print(('SKIP: Skipping \"%s\" dataset from index \"%s\". No S3 URLs to update' % ( + updated_doc['_id'], updated_doc['_index']))) diff --git a/scripts/republish_s3_products.py b/scripts/republish_s3_products.py index 537065e..0ca4dce 100755 --- a/scripts/republish_s3_products.py +++ b/scripts/republish_s3_products.py @@ -1,5 +1,14 @@ #!/usr/bin/env python -import json, requests, sys, os +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys +import os from boto.s3.connection import S3Connection from boto.s3.key import Key @@ -32,9 +41,10 @@ "query": "\"%s\"" % bucket_name } }, - "fields": [ "_id", "urls" ] + "fields": ["_id", "urls"] } -r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, src), data=json.dumps(query)) +r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, src), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] @@ -43,15 +53,18 @@ r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: doc = hit['fields'] - prefix = "%s/" % doc['urls'][0].replace('http://%s.s3-website-%s.amazonaws.com/' % (bucket_name, region), '') - print doc['_id'], prefix + prefix = "%s/" % doc['urls'][0].replace( + 'http://%s.s3-website-%s.amazonaws.com/' % (bucket_name, region), '') + print((doc['_id'], prefix)) localize_urls = [] for i in bucket.list(prefix): #localize_urls.append({ 'url': 's3://%s/%s' % (bucket_name, i.name), 'local_path': '%s/' % os.path.basename(prefix[0:-1]) }) - localize_urls.append({ 'url': 'http://%s.s3-website-%s.amazonaws.com/%s' % (bucket_name, region, i.name), 'local_path': '%s/' % os.path.basename(prefix[0:-1]) }) + localize_urls.append({'url': 'http://%s.s3-website-%s.amazonaws.com/%s' % ( + bucket_name, region, i.name), 'local_path': '%s/' % os.path.basename(prefix[0:-1])}) payload = { "job_type": "job:ingest_dataset", "payload": { @@ -59,6 +72,6 @@ "dataset_urls": localize_urls } } - #print json.dumps(payload, indent=2) + # print json.dumps(payload, indent=2) submit_job.apply_async((payload,), queue="jobs_processed") - #sys.exit() + # sys.exit() diff --git a/scripts/rm_io_users.py b/scripts/rm_io_users.py index f279da8..2ed3722 100755 --- a/scripts/rm_io_users.py +++ b/scripts/rm_io_users.py @@ -1,6 +1,17 @@ #!/usr/bin/env python + from __future__ import print_function -import json, requests, sys, traceback +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import input +from builtins import str +from future import standard_library +standard_library.install_aliases() +import json +import requests +import sys +import traceback import re import fnmatch @@ -8,39 +19,44 @@ USER_RE = re.compile("^([a-zA-Z0-9-_.@]+\s*)+$") es_url = app.config['ES_URL'] + + def get_matching_hysdsios(): # get connection and create destination index # index all docs from source index to destination index query = { - "fields": "_source", - "query": { - "match_all": {} - }, - "sort": [{"_id":{"order":"asc"}}] + "fields": "_source", + "query": { + "match_all": {} + }, + "sort": [{"_id": {"order": "asc"}}] } - r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % (es_url, "hysds_ios"), data=json.dumps(query)) + r = requests.post('%s/%s/_search?search_type=scan&scroll=60m&size=100' % + (es_url, "hysds_ios"), data=json.dumps(query)) scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] users = {} print("Available HySDS IOs:") while True: - r = requests.post('%s/_search/scroll?scroll=60m' % es_url, data=scroll_id) + r = requests.post('%s/_search/scroll?scroll=60m' % + es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] - if len(res['hits']['hits']) == 0: break + if len(res['hits']['hits']) == 0: + break for hit in res['hits']['hits']: ident = hit["_id"] - accounts = hit.get("_source",{}).get("allowed_accounts",[]) + accounts = hit.get("_source", {}).get("allowed_accounts", []) print("\t{0}: {1}".format(ident, json.dumps(accounts))) users[ident] = accounts - #Enter in valid HySDS - IO + # Enter in valid HySDS - IO print("-----------") hysdsios = [] while len(hysdsios) == 0: - pattern = raw_input("Enter in hysds-io id glob pattern (from above):") - hysdsios = fnmatch.filter(users.keys(),pattern) + pattern = eval(input("Enter in hysds-io id glob pattern (from above):")) + hysdsios = fnmatch.filter(list(users.keys()), pattern) if len(hysdsios) == 0: print("No matching hysds-ios for '{0}'".format(pattern)) print() @@ -48,20 +64,25 @@ def get_matching_hysdsios(): print("Selected hysds-io(s): {0}".format(json.dumps(hysdsios))) print("----------") return (hysdsios, users) + + def get_rm_users(): - #User input + # User input user = "" while not USER_RE.match(user): if user != "": print("Invalid username(s): '{0}'".format(user)) - user = raw_input("Enter in space-separated user(s) to remove:") + user = eval(input("Enter in space-separated user(s) to remove:")) user = user.split() return user + + def rm_to_ios(users, hysdsio, existing): ''' ''' if set(existing[hysdsio]) - set(users) == set(existing[hysdsio]): - print("Supplied users don't have permission to run {0}, nothing to do, Skipping.".format(hysdsio)) + print("Supplied users don't have permission to run {0}, nothing to do, Skipping.".format( + hysdsio)) return existing[hysdsio] = list(set(existing[hysdsio]) - set(users)) print() @@ -69,7 +90,8 @@ def rm_to_ios(users, hysdsio, existing): print("------------") sure = "" while sure == "": - sure = raw_input("Are you sure you want to remove {0} to allowed users for '{1}' for a final user set of {2}?" .format(users,hysdsio,json.dumps(existing[hysdsio]))) + sure = eval(input("Are you sure you want to remove {0} to allowed users for '{1}' for a final user set of {2}?" .format( + users, hysdsio, json.dumps(existing[hysdsio])))) if not sure.startswith("y") and sure != "": print("User showed weakness, skipping") return @@ -77,14 +99,18 @@ def rm_to_ios(users, hysdsio, existing): "doc": { "allowed_accounts": existing[hysdsio] }, - "doc_as_upsert":True + "doc_as_upsert": True } try: - r = requests.post('%s/%s/%s/%s/_update' % (es_url, "hysds_ios","hysds_io",hysdsio), data=json.dumps(doc)) + r = requests.post('%s/%s/%s/%s/_update' % (es_url, + "hysds_ios", "hysds_io", hysdsio), data=json.dumps(doc)) r.raise_for_status() except Exception as e: - print("[ERROR] Failed to update hysds-io. Resolve IMMEDIATELY. {0}:{1}\n{2}".format(type(e), str(e), traceback.format_exc()), file=sys.stderr) + print("[ERROR] Failed to update hysds-io. Resolve IMMEDIATELY. {0}:{1}\n{2}".format( + type(e), str(e), traceback.format_exc()), file=sys.stderr) sys.exit(-1) + + if __name__ == "__main__": hysdsios, existing = get_matching_hysdsios() users = get_rm_users() diff --git a/setup.py b/setup.py index 241f140..43b3dd7 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,18 @@ +from __future__ import unicode_literals +from __future__ import print_function +from __future__ import division +from __future__ import absolute_import from setuptools import setup, find_packages setup( name='grq2', - version='1.0.3', + version='1.1.0', long_description='GeoRegionQuery REST API using ElasticSearch backend', packages=find_packages(), include_package_data=True, zip_safe=False, - install_requires=['Flask', 'gunicorn', 'eventlet', 'supervisor', 'pymongo', + install_requires=['Flask', 'gunicorn', 'eventlet', 'pymongo', 'elasticsearch>=1.0.0,<2.0.0', 'requests', 'pyshp', - 'shapely==1.5.15', 'Cython', 'Cartopy==0.13.1', 'redis', - 'flask-restplus>=0.9.2'] + 'shapely>=1.5.15', 'Cython>=0.15.1', 'Cartopy>=0.13.1', + 'redis', 'flask-restplus>=0.9.2', 'future>=0.17.1'] ) diff --git a/test/test_geonames.py b/test/test_geonames.py index b7fa046..b0051c9 100755 --- a/test/test_geonames.py +++ b/test/test_geonames.py @@ -1,5 +1,14 @@ #!/usr/bin/env python -import os, sys, json +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import +from builtins import open +from future import standard_library +standard_library.install_aliases() +import os +import sys +import json from pprint import pprint from grq2.lib.geonames import get_cities, get_continents @@ -11,7 +20,7 @@ def main(): with open('output.json') as f: m = json.load(f) coords = m['metadata']['location']['coordinates'] - lon, lat = m['metadata']['center']['coordinates'] + lon, lat = m['metadata']['center']['coordinates'] cities = get_cities(coords, pop_th=0) continents = get_continents(lon, lat) if len(continents) > 0: @@ -19,8 +28,8 @@ def main(): else: continent = None pprint(cities) - print continent - print get_center(coords) + print(continent) + print((get_center(coords))) if __name__ == "__main__":