Merge branch 'master' into add_persistence

Author: matheuscmelo

README.md

@@ -58,8 +58,3 @@ Endpoints are avaliable on [restapi-endpoints.md](docs/restapi-endpoints.md) doc
 
 ## Avaliable plugins
 * [Kube Jobs](docs/plugins/kubejobs.md)
-* [Spark Sahara](docs/plugins/spark_sahara.md)
-* [Spark Mesos](docs/plugins/spark_mesos.md)
-* [Openstack Generic](docs/plugins/openstack_generic.md)
-* [Web Application](docs/plugins/web_app.md)
-* [External API](docs/plugins/external-api.md)

docs/plugins/external-api.md

@@ -45,5 +45,5 @@ def stop_monitoring(app_id, data):
 def install_plugin(data):
     plugin = data.get('plugin_source')
     source = data.get('install_source')
-    response, status = api.install_plugin(plugin, source)
+    response, status = api.install_plugin(source, plugin)
     return jsonify(response), status

docs/plugins/openstack_generic.md

@@ -16,10 +16,10 @@
 from flask import Flask
 from monitor.api.v10 import rest
 from monitor.service import api
-from monitor.utils.logger import configure_logging
+from monitor.utils import logger
 
 def main():
     app = Flask(__name__)
-    configure_logging()
     app.register_blueprint(rest)
+    logger.configure_logging()
     app.run(host='0.0.0.0', port=api.port, debug=True)

docs/plugins/spark_mesos.md

@@ -18,8 +18,8 @@
 import uuid
 
 
-class SaharaException(Exception):
-    """Base Exception for the Sahara Plugin
+class GenericException(Exception):
+    """Base Exception for the Generic Plugin
 
     To correctly use this class, inherit from it and define
     a 'message' and 'code' properties.
@@ -43,11 +43,11 @@ def __init__(self, message=None, code=None, inject_error_id=True):
             self.message = (('%(message)s\nError ID: %(id)s')
                             % {'message': self.message, 'id': self.uuid})
 
-        super(SaharaException, self).__init__(
+        super(GenericException, self).__init__(
             '%s: %s' % (self.code, self.message))
 
 
-class NotFoundException(SaharaException):
+class NotFoundException(GenericException):
     code = "NOT_FOUND"
     message_template = "Object '%s' is not found"
 
@@ -62,7 +62,7 @@ def __init__(self, value, message_template=None):
         super(NotFoundException, self).__init__(formatted_message)
 
 
-class RemoteCommandException(SaharaException):
+class RemoteCommandException(GenericException):
     code = "REMOTE_COMMAND_FAILED"
     message_template = "Error during command execution: \"%s\""
 
@@ -93,7 +93,7 @@ def to_printable(s):
         super(RemoteCommandException, self).__init__(formatted_message)
 
 
-class TimeoutException(SaharaException):
+class TimeoutException(GenericException):
     code = "TIMEOUT"
     message_template = ("'%(operation)s' timed out after %(timeout)i "
                         "second(s)")
@@ -115,7 +115,7 @@ def __init__(self, timeout, op_name=None, timeout_name=None):
         super(TimeoutException, self).__init__(formatted_message)
 
 
-class Forbidden(SaharaException):
+class Forbidden(GenericException):
     code = "FORBIDDEN"
     message = ("You are not authorized to complete this action")
 
@@ -133,7 +133,7 @@ def __init__(self, message="Malformed message body"):
         self.message = message
 
 
-class MalformedRequestBody(SaharaException):
+class MalformedRequestBody(GenericException):
     code = "MALFORMED_REQUEST_BODY"
     message_template = ("Malformed message body: %(reason)s")
 
@@ -142,7 +142,7 @@ def __init__(self, reason):
         super(MalformedRequestBody, self).__init__(formatted_message)
 
 
-class MaxRetriesExceeded(SaharaException):
+class MaxRetriesExceeded(GenericException):
     code = "MAX_RETRIES_EXCEEDED"
     message_template = ("Operation %(operation)s wasn't executed correctly "
                         "after %(attempts)d attempts")
@@ -154,11 +154,11 @@ def __init__(self, attempts, operation):
         super(MaxRetriesExceeded, self).__init__(formatted_message)
 
 
-class ClusterNotCreatedException(SaharaException):
+class ClusterNotCreatedException(GenericException):
     code = "CLUSTER_NOT_CREATED"
     message = "Cluster could not be created"
 
 
-class ConfigurationError(SaharaException):
+class ConfigurationError(GenericException):
     code = "CONFIGURATION_ERROR"
     message = "The configuration has failed"

docs/plugins/spark_sahara.md

@@ -12,3 +12,24 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from monitor import exceptions as ex
+from monitor.service import api
+from monitor.plugins.kubejobs.plugin import KubeJobProgress
+
+
+class MonitorBuilder:
+    def __init__(self):
+        pass
+
+    def get_monitor(self, plugin, app_id, plugin_info):
+        executor = None
+
+        if plugin == "kubejobs":
+            executor = KubeJobProgress(
+                app_id, plugin_info, retries=api.retries)
+
+        else:
+            raise ex.BadRequestException()
+
+        return executor

docs/plugins/web_app.md

@@ -163,7 +163,6 @@ def monitoring_application(self):
 
     def validate(self, data):
         data_model = {
-            "datasource_type": six.string_types,
             "enable_visualizer": bool,
             "expected_time": int,
             "number_of_jobs": int,
@@ -172,6 +171,9 @@ def validate(self, data):
             "submission_time": six.string_types
         }
 
+        if 'enable_visualizer' in data and data['enable_visualizer']:
+            data_model.update({"datasource_type": six.string_types})
+
         for key in data_model:
             if (key not in data):
                 raise ex.BadRequestException(