Skip to content

Commit 10b24f2

Browse files
committed
update sbt to 1.2.0, update plugin deps, re-sync sbt syntax
1 parent a0282e0 commit 10b24f2

File tree

5 files changed

+65
-58
lines changed

5 files changed

+65
-58
lines changed

README.md

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,18 @@ https://isarn.github.io/isarn-sketches-spark/latest/api/#org.isarnproject.sketch
99
``` scala
1010
// Note that the version of spark and python is part of the release name.
1111
// This example is for spark 2.2 and python 2.7:
12-
libraryDependencies += "org.isarnproject" %% "isarn-sketches-spark" % "0.2.0-sp2.2-py2.7"
12+
libraryDependencies += "org.isarnproject" %% "isarn-sketches-spark" % "0.3.1-sp2.2-py2.7"
13+
```
14+
15+
** Currently supported: python 2.7, 3.6 X spark 2.2, 2.3 X scala 2.11 **
1316

14-
// currently supported: python 2.7, 3.5, spark 2.1, 2.2, scala 2.10, 2.11
17+
If you are interested in a python/spark/scala build that is not listed above, please contact me and/or file an issue!
18+
19+
This package builds against some `% Provided` Apache Spark dependencies:
20+
```scala
21+
libraryDependencies += "org.apache.spark" %% "spark-core" % sparkVersion
22+
libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion
23+
libraryDependencies += "org.apache.spark" %% "spark-mllib" % sparkVersion
1524
```
1625

1726
## Examples

build.sbt

Lines changed: 44 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
1+
// xsbt clean unidoc previewSite
2+
// xsbt clean unidoc ghpagesPushSite
3+
// xsbt -Dsbt.global.base=/home/eje/.sbt/sonatype +publish
4+
// make sure sparkVersion and pythonVersion are set as you want them prior to +publish
5+
6+
import scala.sys.process._
7+
18
name := "isarn-sketches-spark"
29

310
organization := "org.isarnproject"
411

5-
bintrayOrganization := Some("isarn")
12+
val packageVersion = "0.3.1"
613

7-
val packageVersion = "0.4.0-SNAPSHOT"
8-
9-
val sparkVersion = "2.2.0"
14+
val sparkVersion = "2.2.2"
1015

1116
val pythonVersion = "2.7"
1217

@@ -18,11 +23,9 @@ val pythonCMD = s"""python${pythonVersion.split('.').head}"""
1823

1924
version := s"${packageVersion}-${sparkSuffix}-${pythonSuffix}"
2025

21-
scalaVersion := "2.11.8"
22-
23-
crossScalaVersions := Seq("2.10.6", "2.11.8")
26+
scalaVersion := "2.11.12"
2427

25-
useGpg := true
28+
crossScalaVersions := Seq("2.11.12") // scala 2.12 when spark supports it
2629

2730
pomIncludeRepository := { _ => false }
2831

@@ -56,36 +59,34 @@ developers := List(
5659
)
5760
)
5861

59-
def commonSettings = Seq(
60-
libraryDependencies ++= Seq(
61-
"org.isarnproject" %% "isarn-sketches" % "0.1.1",
62-
"org.apache.spark" %% "spark-core" % sparkVersion % Provided,
63-
"org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
64-
"org.apache.spark" %% "spark-mllib" % sparkVersion % Provided,
65-
"org.isarnproject" %% "isarn-scalatest" % "0.0.2" % Test,
66-
"org.scalatest" %% "scalatest" % "2.2.4" % Test,
67-
"org.apache.commons" % "commons-math3" % "3.6.1" % Test),
68-
initialCommands in console := """
69-
|import org.apache.spark.SparkConf
70-
|import org.apache.spark.SparkContext
71-
|import org.apache.spark.sql.SparkSession
72-
|import org.apache.spark.SparkContext._
73-
|import org.apache.spark.rdd.RDD
74-
|import org.apache.spark.ml.linalg.Vectors
75-
|import org.isarnproject.sketches.TDigest
76-
|import org.isarnproject.sketches.udaf._
77-
|import org.apache.spark.isarnproject.sketches.udt._
78-
|val initialConf = new SparkConf().setAppName("repl").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer").set("spark.kryoserializer.buffer", "16mb")
79-
|val spark = SparkSession.builder.config(initialConf).master("local[2]").getOrCreate()
80-
|import spark._, spark.implicits._
81-
|val sc = spark.sparkContext
82-
|import org.apache.log4j.{Logger, ConsoleAppender, Level}
83-
|Logger.getRootLogger().getAppender("console").asInstanceOf[ConsoleAppender].setThreshold(Level.WARN)
84-
""".stripMargin,
85-
cleanupCommands in console := "spark.stop"
86-
)
87-
88-
seq(commonSettings:_*)
62+
libraryDependencies ++= Seq(
63+
"org.isarnproject" %% "isarn-sketches" % "0.1.2",
64+
"org.apache.spark" %% "spark-core" % sparkVersion % Provided,
65+
"org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
66+
"org.apache.spark" %% "spark-mllib" % sparkVersion % Provided,
67+
"org.isarnproject" %% "isarn-scalatest" % "0.0.3" % Test,
68+
"org.scalatest" %% "scalatest" % "3.0.5" % Test,
69+
"org.apache.commons" % "commons-math3" % "3.6.1" % Test)
70+
71+
initialCommands in console := """
72+
|import org.apache.spark.SparkConf
73+
|import org.apache.spark.SparkContext
74+
|import org.apache.spark.sql.SparkSession
75+
|import org.apache.spark.SparkContext._
76+
|import org.apache.spark.rdd.RDD
77+
|import org.apache.spark.ml.linalg.Vectors
78+
|import org.isarnproject.sketches.TDigest
79+
|import org.isarnproject.sketches.udaf._
80+
|import org.apache.spark.isarnproject.sketches.udt._
81+
|val initialConf = new SparkConf().setAppName("repl").set("spark.serializer", "org.apache.spark.serializer.KryoSerializer").set("spark.kryoserializer.buffer", "16mb")
82+
|val spark = SparkSession.builder.config(initialConf).master("local[2]").getOrCreate()
83+
|import spark._, spark.implicits._
84+
|val sc = spark.sparkContext
85+
|import org.apache.log4j.{Logger, ConsoleAppender, Level}
86+
|Logger.getRootLogger().getAppender("console").asInstanceOf[ConsoleAppender].setThreshold(Level.WARN)
87+
""".stripMargin
88+
89+
cleanupCommands in console := "spark.stop"
8990

9091
licenses += ("Apache-2.0", url("http://opensource.org/licenses/Apache-2.0"))
9192

@@ -118,9 +119,9 @@ compilePython := {
118119
}
119120
}
120121

121-
compilePython <<= compilePython.dependsOn(deletePYC)
122+
compilePython := (compilePython.dependsOn(deletePYC)).value
122123

123-
(packageBin in Compile) <<= (packageBin in Compile).dependsOn(compilePython)
124+
(packageBin in Compile) := ((packageBin in Compile).dependsOn(compilePython)).value
124125

125126
mappings in (Compile, packageBin) ++= Seq(
126127
(baseDirectory.value / "python" / "isarnproject" / "__init__.pyc") -> "isarnproject/__init__.pyc",
@@ -142,13 +143,10 @@ assemblyShadeRules in assembly := Seq(
142143

143144
scalacOptions in (Compile, doc) ++= Seq("-doc-root-content", baseDirectory.value+"/root-doc.txt")
144145

145-
site.settings
146-
147-
site.includeScaladoc()
146+
enablePlugins(ScalaUnidocPlugin, GhpagesPlugin)
148147

149-
// Re-enable if/when we want to support gh-pages w/ jekyll
150-
// site.jekyllSupport()
148+
siteSubdirName in ScalaUnidoc := "latest/api"
151149

152-
ghpages.settings
150+
addMappingsToSiteDir(mappings in (ScalaUnidoc, packageDoc), siteSubdirName in ScalaUnidoc)
153151

154152
git.remoteRepo := "git@github.com:isarn/isarn-sketches-spark.git"

project/assembly.sbt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4")
1+
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.7")

project/build.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
sbt.version=0.13.11
1+
sbt.version=1.2.0

project/plugins.sbt

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@ resolvers += "sonatype-releases" at "https://oss.sonatype.org/content/repositori
77

88
resolvers += "jgit-repo" at "http://download.eclipse.org/jgit/maven"
99

10-
addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0")
10+
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.2")
1111

12-
addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.4")
12+
addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1")
1313

14-
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.6.0")
14+
addSbtPlugin("io.crashbox" % "sbt-gpg" % "0.1.6")
1515

16-
addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.1")
17-
18-
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.0")
16+
addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "2.3")
1917

2018
// scoverage and coveralls deps are at old versions to avoid a bug in the current versions
2119
// update these when this fix is released: https://github.com/scoverage/sbt-coveralls/issues/73
22-
addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4")
20+
//addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4")
21+
22+
//addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0")
2323

24-
addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0")
24+
//addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.6.0")

0 commit comments

Comments
 (0)