LIVY-228. Shade json4s and py4j in repl.

Version of dependencies are different between Spark 1 and Spark 2 (e.g. json4s, py4j). Shade them to make supporting multiple versions simultaneously easier.

Closes #207

Author: jerryshao

pom.xml

@@ -985,8 +985,6 @@
       </activation>
       <properties>
         <spark.version>1.6.2</spark.version>
-        <py4j.version>0.9</py4j.version>
-        <json4s.version>3.2.10</json4s.version>
       </properties>
     </profile>
 
@@ -999,8 +997,6 @@
       </activation>
       <properties>
         <spark.version>2.0.0</spark.version>
-        <py4j.version>0.10.1</py4j.version>
-        <json4s.version>3.2.11</json4s.version>
       </properties>
     </profile>
   </profiles>

repl/pom.xml

@@ -116,19 +116,16 @@
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-ast_${scala.binary.version}</artifactId>
-            <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-core_${scala.binary.version}</artifactId>
-            <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-jackson_${scala.binary.version}</artifactId>
-            <scope>provided</scope>
         </dependency>
 
         <dependency>
@@ -165,6 +162,46 @@
 
     <build>
         <plugins>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>shade</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <shadedArtifactAttached>false</shadedArtifactAttached>
+                            <artifactSet>
+                                <includes>
+                                    <include>org.json4s:json4s-ast_${scala.binary.version}</include>
+                                    <include>org.json4s:json4s-core_${scala.binary.version}</include>
+                                    <include>org.json4s:json4s-jackson_${scala.binary.version}</include>
+                                </includes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>*.jar</exclude>
+                                        <exclude>META-INF/maven/**</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <relocations>
+                                <relocation>
+                                    <pattern>org.json4s</pattern>
+                                    <shadedPattern>com.cloudera.livy.shaded.json4s</shadedPattern>
+                                </relocation>
+                            </relocations>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-dependency-plugin</artifactId>
@@ -174,6 +211,15 @@
                         <goals>
                             <goal>copy-dependencies</goal>
                         </goals>
+                        <configuration>
+                            <excludeArtifactIds>
+                                json4s-ast_${scala.binary.version},
+                                json4s-core_${scala.binary.version},
+                                json4s-jackson_${scala.binary.version},
+                                paranamer,
+                                scalap
+                            </excludeArtifactIds>
+                        </configuration>
                     </execution>
                 </executions>
             </plugin>

repl/src/main/scala/com/cloudera/livy/repl/PythonInterpreter.scala

@@ -276,9 +276,13 @@ private class PythonInterpreter(process: Process, gatewayServer: GatewayServer,
 
   private def updatePythonGatewayPort(port: Int): Unit = {
     // The python gateway port can be 0 only when LivyConf.TEST_MODE is true
+    // Py4j 0.10 has different API signature for "getCallbackClient", use reflection to handle it.
     if (port != 0) {
-      val callbackClient = gatewayServer.getCallbackClient
-      val field = callbackClient.getClass.getDeclaredField("port")
+      val callbackClient = gatewayServer.getClass
+        .getMethod("getCallbackClient")
+        .invoke(gatewayServer)
+
+      val field = Class.forName("py4j.CallbackClient").getDeclaredField("port")
       field.setAccessible(true)
       field.setInt(callbackClient, port.toInt)
     }