Skip to content

Commit

Permalink
HC-476: Make user dataset rule evaluation more efficient/targetted (#62)
Browse files Browse the repository at this point in the history 
* added index_pattern to user rules

* using validate_index_pattern from hysds.utils
  • Loading branch information
@DustinKLo
DustinKLo authored Jul 12, 2023
1 parent 861f780 commit 10dd330
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 1 deletion.
3 changes: 3 additions & 0 deletions config/user_rules_dataset.mapping
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@
"query_string": {
"type": "text"
},
"index_pattern": {
"type": "keyword"
},
"queue": {
"type": "text"
},
Expand Down
26 changes: 26 additions & 0 deletions grq2/services/api_v01/user_rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from flask_restx import Resource, inputs

from hysds.celery import app as celery_app
from hysds.utils import validate_index_pattern
from hysds_commons.action_utils import check_passthrough_query

from grq2 import app, mozart_es
Expand All @@ -38,6 +39,7 @@ class UserRules(Resource):
post_parser.add_argument('hysds_io', type=str, required=True, location='form', help='hysds io')
post_parser.add_argument('job_spec', type=str, required=True, location='form', help='queue')
post_parser.add_argument('priority', type=int, required=True, location='form', help='RabbitMQ job priority (0-9)')
post_parser.add_argument('index_pattern', type=str, required=True, location='form', help='ES index pattern')
post_parser.add_argument('query_string', type=str, required=True, location='form', help='elasticsearch query')
post_parser.add_argument('kwargs', type=str, required=True, location='form', help='keyword arguments for PGE')
post_parser.add_argument('queue', type=str, required=True, location='form', help='RabbitMQ job queue')
Expand All @@ -53,6 +55,7 @@ class UserRules(Resource):
put_parser.add_argument('hysds_io', type=str, location='form', help='hysds io')
put_parser.add_argument('job_spec', type=str, location='form', help='queue')
put_parser.add_argument('priority', type=int, location='form', help='RabbitMQ job priority (0-9)')
put_parser.add_argument('index_pattern', type=str, required=True, location='form', help='ES index pattern')
put_parser.add_argument('query_string', type=str, location='form', help='elasticsearch query')
put_parser.add_argument('kwargs', type=str, location='form', help='keyword arguments for PGE')
put_parser.add_argument('queue', type=str, location='form', help='RabbitMQ job queue')
Expand Down Expand Up @@ -119,6 +122,7 @@ def post(self):
job_spec = request_data.get('job_spec')
priority = int(request_data.get('priority', 0))
query_string = request_data.get('query_string')
index_pattern = request_data.get('index_pattern', "").strip()
kwargs = request_data.get('kwargs', '{}')
queue = request_data.get('queue')
tags = request_data.get('tags', [])
Expand Down Expand Up @@ -217,6 +221,15 @@ def post(self):
"tags": tags
}

if not validate_index_pattern(index_pattern):
return {
'success': False,
'message': "index pattern is too broad"
}, 400

if index_pattern:
new_doc["index_pattern"] = index_pattern

if time_limit and isinstance(time_limit, int):
if time_limit <= 0 or time_limit > 86400 * 7:
return {
Expand Down Expand Up @@ -264,6 +277,7 @@ def put(self): # TODO: add user role and permissions
job_spec = request_data.get('job_spec')
priority = request_data.get('priority')
query_string = request_data.get('query_string')
index_pattern = request_data.get('index_pattern', "").strip()
kwargs = request_data.get('kwargs')
queue = request_data.get('queue')
enabled = request_data.get('enabled')
Expand Down Expand Up @@ -333,6 +347,18 @@ def put(self): # TODO: add user role and permissions
'success': False,
'message': 'invalid elasticsearch query JSON'
}, 400

if "index_pattern" in request_data:
if not validate_index_pattern(index_pattern):
return {
'success': False,
'message': "index pattern is too broad"
}, 400
if index_pattern:
update_doc["index_pattern"] = index_pattern # noqa
else:
update_doc["index_pattern"] = None

if kwargs:
update_doc['kwargs'] = kwargs
try:
Expand Down
26 changes: 26 additions & 0 deletions grq2/services/api_v02/user_rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from flask_restx import Resource, inputs

from hysds.celery import app as celery_app
from hysds.utils import validate_index_pattern
from hysds_commons.action_utils import check_passthrough_query

from grq2 import app, mozart_es
Expand All @@ -39,6 +40,7 @@ class UserRules(Resource):
post_parser.add_argument('job_spec', type=str, required=True, location='form', help='queue')
post_parser.add_argument('priority', type=int, required=True, location='form', help='RabbitMQ job priority (0-9)')
post_parser.add_argument('query_string', type=str, required=True, location='form', help='elasticsearch query')
post_parser.add_argument('index_pattern', type=str, required=True, location='form', help='ES index pattern')
post_parser.add_argument('kwargs', type=str, required=True, location='form', help='keyword arguments for PGE')
post_parser.add_argument('queue', type=str, required=True, location='form', help='RabbitMQ job queue')
post_parser.add_argument('tags', type=list, location='form', help='user defined tags for trigger rule')
Expand All @@ -54,6 +56,7 @@ class UserRules(Resource):
put_parser.add_argument('job_spec', type=str, location='form', help='queue')
put_parser.add_argument('priority', type=int, location='form', help='RabbitMQ job priority (0-9)')
put_parser.add_argument('query_string', type=str, location='form', help='elasticsearch query')
post_parser.add_argument('index_pattern', type=str, required=True, location='form', help='ES index pattern')
put_parser.add_argument('kwargs', type=str, location='form', help='keyword arguments for PGE')
put_parser.add_argument('queue', type=str, location='form', help='RabbitMQ job queue')
put_parser.add_argument('tags', type=list, location='form', help='user defined tags for trigger rule')
Expand Down Expand Up @@ -119,6 +122,7 @@ def post(self):
job_spec = request_data.get('job_spec')
priority = int(request_data.get('priority', 0))
query_string = request_data.get('query_string')
index_pattern = request_data.get('index_pattern', "").strip()
kwargs = request_data.get('kwargs', '{}')
queue = request_data.get('queue')
tags = request_data.get('tags', [])
Expand Down Expand Up @@ -217,6 +221,15 @@ def post(self):
"tags": tags
}

if not validate_index_pattern(index_pattern):
return {
'success': False,
'message': "index pattern is too broad"
}, 400

if index_pattern:
new_doc["index_pattern"] = index_pattern

if time_limit and isinstance(time_limit, int):
if time_limit <= 0 or time_limit > 86400 * 7:
return {
Expand Down Expand Up @@ -264,6 +277,7 @@ def put(self): # TODO: add user role and permissions
job_spec = request_data.get('job_spec')
priority = request_data.get('priority')
query_string = request_data.get('query_string')
index_pattern = request_data.get('index_pattern', "").strip()
kwargs = request_data.get('kwargs')
queue = request_data.get('queue')
enabled = request_data.get('enabled')
Expand Down Expand Up @@ -333,6 +347,18 @@ def put(self): # TODO: add user role and permissions
'success': False,
'message': 'invalid elasticsearch query JSON'
}, 400

if "index_pattern" in request_data:
if not validate_index_pattern(index_pattern):
return {
'success': False,
'message': "index pattern is too broad"
}, 400
if index_pattern:
update_doc["index_pattern"] = index_pattern # noqa
else:
update_doc["index_pattern"] = None

if kwargs:
update_doc['kwargs'] = kwargs
try:
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

setup(
name='grq2',
version='2.0.23',
version='2.0.24',
long_description='GeoRegionQuery REST API using ElasticSearch backend',
packages=find_packages(),
include_package_data=True,
Expand Down

0 comments on commit 10dd330

Please sign in to comment.