Remove pyspark (use one in SPARK_HOME)

Author: Jaroslaw Osmanski

seahorse-workflow-executor/workflowexecutor/src/main/scala/ai/deepsense/workflowexecutor/WorkflowExecutorApp.scala

@@ -243,7 +243,7 @@ object WorkflowExecutorApp extends Logging {
       ).execute()
     } else {
       // Running in non-interactive mode
-      val pythonPathGenerator = new pyspark.Loader(None).load
+      val pythonPathGenerator = pyspark.Loader.load
         .map(new PythonPathGenerator(_))
         .getOrElse(throw new RuntimeException("Could not find PySpark!"))
 

seahorse-workflow-executor/workflowexecutor/src/main/scala/ai/deepsense/workflowexecutor/executor/SessionExecutor.scala

@@ -16,7 +16,6 @@
 
 package ai.deepsense.workflowexecutor.executor
 
-import java.io.File
 import java.net.{InetAddress, URL}
 
 import scala.concurrent.duration._
@@ -102,7 +101,7 @@ case class SessionExecutor(
 
     val tempPath = Unzip.unzipAll(depsZip)
 
-    val pythonPathGenerator = new pyspark.Loader(Some(tempPath)).load
+    val pythonPathGenerator = pyspark.Loader.load
       .map(new PythonPathGenerator(_))
       .getOrElse(throw new RuntimeException("Could not find PySpark!"))
 

seahorse-workflow-executor/workflowexecutor/src/main/scala/ai/deepsense/workflowexecutor/pyspark/Loader.scala

@@ -20,17 +20,9 @@ import scala.reflect.io.Path
 
 import ai.deepsense.commons.utils.Logging
 
-class Loader(
-  private val localPath: Option[String] = None
-) extends Logging {
+object Loader extends Logging {
 
-  def load: Option[String] = {
-    localPath match {
-      case Some(x) => Some(x)
-      case None =>
-        Option(System.getenv("SPARK_HOME")).map(pysparkPath)
-    }
-  }
+  def load: Option[String] = Option(System.getenv("SPARK_HOME")).map(pysparkPath)
 
   private def pysparkPath(sparkHome: String): String = {
     val path = Path(sparkHome)./("python").toAbsolute.toString()