Skip to content

Commit 33b20f5

Browse files
OferEalex-the-man
authored andcommitted
LIVY-220. Workaround SPARK-17512 for Spark 2.0.0.
SparkConf "spark.submit.pyFiles" doesn't work in Spark 2.0.0. Use Spark argument "--py-files" instead to workaround the Spark bug. https://issues.apache.org/jira/browse/SPARK-17512 Closes #197
1 parent eeb6d01 commit 33b20f5

File tree

1 file changed

+7
-2
lines changed

1 file changed

+7
-2
lines changed

server/src/main/scala/com/cloudera/livy/utils/SparkProcessBuilder.scala

+7-2
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,13 @@ class SparkProcessBuilder(livyConf: LivyConf) extends Logging {
178178
addOpt("--name", _name)
179179
addOpt("--class", _className)
180180
_conf.foreach { case (key, value) =>
181-
arguments += "--conf"
182-
arguments += f"$key=$value"
181+
if (key == "spark.submit.pyFiles") {
182+
arguments += "--py-files"
183+
arguments += f"$value"
184+
} else {
185+
arguments += "--conf"
186+
arguments += f"$key=$value"
187+
}
183188
}
184189
addList("--driver-class-path", _driverClassPath)
185190

0 commit comments

Comments
 (0)