Skip to content

Commit

Permalink
HC-304 (#33)
Browse files Browse the repository at this point in the history 
* removed un-used imports from api v01 & 02

added new rest api endpoint for registering and building jobs in jenkins

* imported service in mozart.__init__.py

* added get api request for EventSource javascript api

TODO: should probably move the build step to a different rest api endpoint
fixed sdscli build command

* changed execute command (using with now) in jenkins.py added logger as well

* removed bufsize=1 from subprocess.Popen

* added event_source option (defaults to False) to execute to be compatible with the EventSource javascript API, which is prepending with 'data' and adding a newline

* re-organized the jenkins rest api and bumped version

* de-coupled api_v01.py into multiple modules (testing, will de-couple v02 if all goes well)

* moved namespaces to mozart/services/api_v01/__init__.py

* de-coupled the namespaces across multiple files

got the swagger UI working
re-organized mozart/__init__.py

* de-coupled API v0.2

moved code from api_v0X/__init__.py to a service.py module to allow for the swagger /doc/ endpoint

* bumped version

removed jenkins.py (branched off from a un-related ticket)
removed trailing newline

* handling global 404 error with a json response instead of the default HTML response

* bumped version too high

* forgot to remove mozart/services/api_v02.py

* accidentally imported v01 classes to v02's service.py

* removed elasticsearch-dsl because its unused

* added endpoint to retrieve products staged from a job

* re-added jenkins.py from previous branch

added python-jenkins to setup.py
JENKINS_[HOST|USER|API_KEY] added to settings.cfg.tmpl
added jenkins connection to mozart/__init__.py

* utilized python-jenkins to build and delete jobs

added functionality to stop builds (including jobs in queued state waiting to be built
removed uses of sdscli in jenkins.py (except for job registration bc job registration is a little more complicated, maybe do it later))
added JENKINS_ENABLED in settings.cfg (name subject to change)

* renamed jenkins.py and its service to ci

* fixed cofig parser for JENKINS_* in settings.cfg

added (argument) parser for JobBuilder, so it shows up on the swagger UI
added additional logging for when job isnt found in Jenkins

* renamed endpoint from jenkins -> ci

* added jenkins job status API endpoint

added optional URL parameters for job build and job status endpoints
will fallback to parsing the Git repo and branch if job_name not supplied in URL parameters

* added input parser to JobRegistration endpoint

* moved conditional jenkins resoruces to __init__.py

so the endpoints wont show up and will return a "404 resource not found" when ENABLE_JENKINS is set to false
added api to remove job builds (note: will need to stop build first before removing)
changed DELETE requests in ci.py to retrieve data from query parameters instead of form data
fixed {{ JENKINS_USER }} TO {{ JENKINS_API_USER }} in settings.cfg

* removed comments in __init__.py

* added endpoint to list all registered jobs

* added parsers for remaining endpoints in api v0,1 and v0.2

* removed duplicate parser argument

* commented out logger.propagate (for now)

re-added request data parser in GetJobInfo (get) to not change too much in api_v01
added url param (_id) to JobInfo in api_v02

* removed example jenkins endpoint from ci.py

* added build_number in query params as a fallback if not provided in url param

* added endpoint to check if job exists in Jenkins

* git status

* fixed build_number query arg in ci.py (Build)

* added python-jenkins to setup.py

* added description in settings.cfg.tmpl
  • Loading branch information
@DustinKLo
DustinKLo authored Feb 9, 2021
1 parent 8f2b611 commit 618bd57
Show file tree
Hide file tree
Showing 15 changed files with 859 additions and 238 deletions.
11 changes: 11 additions & 0 deletions mozart/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from flask_cors import CORS # TODO: will remove this once we figure out the proper host for the UI

from hysds_commons.elasticsearch_utils import ElasticsearchUtility
from jenkins import Jenkins


class ReverseProxied(object):
Expand Down Expand Up @@ -85,6 +86,7 @@ def resource_not_found(e):
app = Flask(__name__)
app.wsgi_app = ReverseProxied(app.wsgi_app)
app.config.from_pyfile('../settings.cfg')
# app.logger.propagate = False # prevents duplicate logging

# TODO: will remove this when ready for actual release, need to figure out the right host
CORS(app)
Expand All @@ -106,6 +108,15 @@ def resource_not_found(e):
# Mozart's connection to Elasticsearch
mozart_es = ElasticsearchUtility(app.config['ES_URL'], app.logger)

# add jenkins connection
if app.config.get('JENKINS_ENABLED', False):
jenkins_wrapper = Jenkins(app.config['JENKINS_HOST'], username=app.config['JENKINS_USER'],
password=app.config['JENKINS_API_KEY'])
from mozart.services.ci import services as ci_services
app.register_blueprint(ci_services)
else:
jenkins_wrapper = None

# views blueprints
from mozart.views.main import mod as views_module
app.register_blueprint(views_module)
Expand Down
96 changes: 38 additions & 58 deletions mozart/services/api_v01/builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,8 @@
class GetJobTypes(Resource):
"""Get list of registered job types and return as JSON."""
resp_model_job_types = job_spec_ns.model('Job Type List Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing " + "success or failure"),
'result': fields.List(fields.String, required=True, description="list of job types")
})
Expand Down Expand Up @@ -68,11 +67,9 @@ class GetJobSpecType(Resource):
"""Get list of job queues and return as JSON."""

resp_model = job_spec_ns.model('Job Type Specification Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.Raw(required=True, description="Job Type Specification")
})
parser = job_spec_ns.parser()
Expand Down Expand Up @@ -108,11 +105,9 @@ class AddJobSpecType(Resource):
"""Add job spec"""

resp_model = job_spec_ns.model('Job Type Specification Addition Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.String(required=True, description="Job Type Specification ID")
})
parser = job_spec_ns.parser()
Expand Down Expand Up @@ -145,13 +140,10 @@ def post(self):
description="Removes a job type specification.")
class RemoveJobSpecType(Resource):
"""Remove job spec"""

resp_model = job_spec_ns.model('Job Type Specification Removal Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
})
parser = job_spec_ns.parser()
parser.add_argument('id', required=True, type=str, help="Job Type Specification ID")
Expand All @@ -171,7 +163,7 @@ def get(self):
app.logger.info('Deleted job_spec %s from index: %s' % (_id, JOB_SPECS_INDEX))
return {
'success': True,
'message': ""
'message': "job spec removed: %s" % _id
}


Expand All @@ -181,13 +173,10 @@ def get(self):
class GetContainerTypes(Resource):
"""Get list of registered containers and return as JSON."""
resp_model_job_types = container_ns.model('Container List Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.List(fields.String, required=True,
description="list of hysds-io types")
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.List(fields.String, required=True, description="list of hysds-io types")
})

@container_ns.marshal_with(resp_model_job_types)
Expand Down Expand Up @@ -259,11 +248,9 @@ class GetContainerRemove(Resource):
"""Remove a container"""

resp_model = container_ns.model('Container Removal Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure")
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure")
})
parser = container_ns.parser()
parser.add_argument('id', required=True, type=str, help="Container ID")
Expand Down Expand Up @@ -294,11 +281,9 @@ class GetContainerInfo(Resource):
"""Info a container"""

resp_model = container_ns.model('Container Info Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.Raw(required=True, description="Container Info")
})
parser = container_ns.parser()
Expand Down Expand Up @@ -330,11 +315,9 @@ def get(self):
class GetHySDSIOTypes(Resource):
"""Get list of registered hysds-io and return as JSON."""
resp_model_job_types = container_ns.model('HySDS IO List Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.List(fields.String, required=True,
description="list of hysds-io types")
})
Expand All @@ -358,11 +341,9 @@ class GetHySDSIOType(Resource):
"""Get list of job queues and return as JSON."""

resp_model = hysds_io_ns.model('HySDS IO Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.Raw(required=True, description="HySDS IO Object")
})
parser = hysds_io_ns.parser()
Expand All @@ -378,7 +359,10 @@ def get(self):

hysds_io = mozart_es.get_by_id(index=HYSDS_IOS_INDEX, id=_id, ignore=404)
if hysds_io['found'] is False:
return {'success': False, 'message': ""}, 404
return {
'success': False,
'message': "hysds io not found: %s" % _id
}, 404

return {
'success': True,
Expand All @@ -393,11 +377,9 @@ class AddHySDSIOType(Resource):
"""Add job spec"""

resp_model = hysds_io_ns.model('HySDS IO Addition Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.String(required=True, description="HySDS IO ID")
})
parser = hysds_io_ns.parser()
Expand Down Expand Up @@ -436,11 +418,9 @@ class RemoveHySDSIOType(Resource):
"""Remove job spec"""

resp_model = hysds_io_ns.model('HySDS IO Removal Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception; "
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
})
parser = hysds_io_ns.parser()
parser.add_argument('id', required=True, type=str, help="HySDS IO ID")
Expand Down
15 changes: 6 additions & 9 deletions mozart/services/api_v01/events.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,9 @@ class AddLogEvent(Resource):
"""Add log event."""

resp_model = event_ns.model('HySDS Event Log Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'success': fields.Boolean(required=True, description="if 'false', encountered exception;"
"otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.String(required=True, description="HySDS custom event log ID")
})
parser = event_ns.parser()
Expand All @@ -49,8 +47,7 @@ class AddLogEvent(Resource):
"reservation": "r-02fd006170749a0a8",
"termination_date": "2015-01-02T15:49:05.571384"
}""")
parser.add_argument('tags', required=False, type=str,
help='JSON list of tags, e.g. ["dumby", "test_job"]')
parser.add_argument('tags', required=False, type=str, help='JSON list of tags, e.g. ["dumby", "test_job"]')
parser.add_argument('hostname', required=False, type=str,
help='Event-related hostname, e.g. "job.hysds.net", "192.168.0.1"')

Expand All @@ -64,8 +61,8 @@ def post(self):
try:
form = json.loads(request.data)
except Exception as e:
raise Exception(
"Failed to parse request data. '{0}' is malformed JSON".format(request.data))
app.logger.error(e)
raise Exception("Failed to parse request data. '{0}' is malformed JSON".format(request.data))
else:
form = request.form

Expand Down
Loading

0 comments on commit 618bd57

Please sign in to comment.