Skip to content

Commit

Permalink
HC-467: Be able to modify a job's disk usage/time limits through job …
Browse files Browse the repository at this point in the history 
…submission (#48)

* added time_limit + disk_usage to job submit API

bump version

* adding validation to time_limit

---------

Co-authored-by: dustinlo <dustin.k.lo@jpl.nasa.gov>
  • Loading branch information
@DustinKLo
DustinKLo and dustinlo authored May 11, 2023
1 parent ec28b64 commit f278eb9
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 6 deletions.
27 changes: 25 additions & 2 deletions mozart/services/api_v01/jobs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ class SubmitJob(Resource):
parser.add_argument('payload_hash', type=str, help='user-generated payload hash')
parser.add_argument('username', type=str, help='user to submit job')
parser.add_argument('enable_dedup', type=bool, help='flag to enable/disable job dedup')
parser.add_argument('soft_time_limit', type=str, help='soft time limit for job execution')
parser.add_argument('time_limit', type=str, help='hard time limit for job execution')
parser.add_argument('disk_usage', type=str, help='disk usage for PGE (KB, MB, GB, etc)')
parser.add_argument('params', type=str,
help="""JSON job context, e.g. {
"entity_id": "LC80101172015002LGN00",
Expand Down Expand Up @@ -223,6 +226,10 @@ def post(self):
payload_hash = request.form.get('payload_hash', request.args.get('payload_hash', None))
enable_dedup = str(request.form.get('enable_dedup', request.args.get('enable_dedup', "true")))

soft_time_limit = request.form.get('soft_time_limit', request.args.get('soft_time_limit', None))
time_limit = request.form.get('time_limit', request.args.get('time_limit', None))
disk_usage = request.form.get('disk_usage', request.args.get('disk_usage', None))

try:
if enable_dedup.strip().lower() == "true":
enable_dedup = True
Expand All @@ -231,6 +238,21 @@ def post(self):
else:
raise Exception("Invalid value for param 'enable_dedup': {0}".format(enable_dedup))

if soft_time_limit is not None:
soft_time_limit = int(soft_time_limit)
if soft_time_limit < 1:
return {
'success': False,
'message': "soft_time_limit must be greater than 0"
}, 400
if time_limit is not None:
time_limit = int(time_limit)
if time_limit < 1:
return {
'success': False,
'message': "time_limit must be greater than 0"
}, 400

try:
if tags is not None:
tags = json.loads(tags)
Expand All @@ -251,13 +273,14 @@ def post(self):
app.logger.warning(job_queue)
job_json = hysds_commons.job_utils.resolve_hysds_job(job_type, job_queue, priority, tags, params,
username=username, job_name=job_name,
payload_hash=payload_hash, enable_dedup=enable_dedup)
payload_hash=payload_hash, enable_dedup=enable_dedup,
soft_time_limit=soft_time_limit, time_limit=time_limit,
disk_usage=disk_usage)
ident = hysds_commons.job_utils.submit_hysds_job(job_json)
except Exception as e:
message = "Failed to submit job. {0}:{1}".format(type(e), str(e))
app.logger.error(message)
return {'success': False, 'message': message}, 500

return {
'success': True,
'message': '',
Expand Down
41 changes: 38 additions & 3 deletions mozart/services/api_v02/jobs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,21 @@ class SubmitJob(Resource):
parser.add_argument('payload_hash', required=False, type=str, help='user-generated payload hash')
parser.add_argument('username', type=str, help='user to submit job')
parser.add_argument('enable_dedup', required=False, type=bool, help='flag to enable/disable job dedup')
parser.add_argument('params', required=False, type=str, help="JSON job context for PGE")
parser.add_argument('soft_time_limit', type=str, help='soft time limit for job execution')
parser.add_argument('time_limit', type=str, help='hard time limit for job execution')
parser.add_argument('disk_usage', type=str, help='disk usage for PGE (KB, MB, GB, etc)')
parser.add_argument('params', required=False, type=str,
help="""JSON job context for PGE, e.g. {
"entity_id": "LC80101172015002LGN00",
"min_lat": -79.09923,
"max_lon": -125.09297,
"id": "dumby-product-20161114180506209624",
"acq_time": "2015-01-02T15:49:05.571384",
"min_sleep": 1,
"max_lat": -77.7544,
"min_lon": -139.66082,
"max_sleep": 10
}""")

@job_ns.marshal_with(resp_model)
@job_ns.expect(parser, validate=True)
Expand All @@ -73,13 +87,33 @@ def post(self):
payload_hash = request.form.get('payload_hash', request.args.get('payload_hash', None))
enable_dedup = str(request.form.get('enable_dedup', request.args.get('enable_dedup', "true")))

soft_time_limit = request.form.get('soft_time_limit', request.args.get('soft_time_limit', None))
time_limit = request.form.get('time_limit', request.args.get('time_limit', None))
disk_usage = request.form.get('disk_usage', request.args.get('disk_usage', None))

try:
if enable_dedup.strip().lower() == "true":
enable_dedup = True
elif enable_dedup.strip().lower() == "false":
enable_dedup = False
else:
raise Exception("Invalid value for param 'enable_dedup': {0}".format(enable_dedup))

if soft_time_limit is not None:
soft_time_limit = int(soft_time_limit)
if soft_time_limit < 1:
return {
'success': False,
'message': "soft_time_limit must be greater than 0"
}, 400
if time_limit is not None:
time_limit = int(time_limit)
if time_limit < 1:
return {
'success': False,
'message': "time_limit must be greater than 0"
}, 400

try:
if tags is not None:
tags = json.loads(tags)
Expand All @@ -100,13 +134,14 @@ def post(self):
app.logger.warning(job_queue)
job_json = hysds_commons.job_utils.resolve_hysds_job(job_type, job_queue, priority, tags, params,
username=username, job_name=job_name,
payload_hash=payload_hash, enable_dedup=enable_dedup)
payload_hash=payload_hash, enable_dedup=enable_dedup,
soft_time_limit=soft_time_limit, time_limit=time_limit,
disk_usage=disk_usage)
ident = hysds_commons.job_utils.submit_hysds_job(job_json)
except Exception as e:
message = "Failed to submit job. {0}:{1}".format(type(e), str(e))
app.logger.error(message)
return {'success': False, 'message': message}, 500

return {
'success': True,
'message': '',
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

setup(
name='mozart',
version='2.2.1',
version='2.2.2',
long_description='HySDS job orchestration/worker web interface',
packages=find_packages(),
include_package_data=True,
Expand Down

0 comments on commit f278eb9

Please sign in to comment.