Skip to content

Commit a586d1c

Browse files
radford1radford1
authoredOct 28, 2020
Added spark profiles to build 2.2/3/4 and 3.0 (#286)
* Added spark profiles to build 2.2/3/4 and eventually 3.0 * Added spark profiles to build 2.2/3/4 and eventually 3.0 Added spark 3 compatibility and cleaned up deprecated code in tests * Added spark profiles to build 2.2/3/4 and eventually 3.0 Added spark 3 compatibility and cleaned up deprecated code in tests Added test jar generation and spark shell testing helper * Added spark profiles to build 2.2/3/4 and eventually 3.0 Added spark 3 compatibility and cleaned up deprecated code in tests Added test jar generation and spark shell testing helper * Fixing code formatting errors from Travis Build * Fixing code formatting errors from Travis Build
1 parent 0a4bc0a commit a586d1c

29 files changed

+351
-178
lines changed
 

‎Makefile

+2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
build:
22
mvn install
3+
mvn clean install -P scala-2.12
34

45
travis-deploy:
56
gpg --import .travis/private-signing-key.gpg
67
mvn versions:set -DnewVersion=${TRAVIS_TAG}
78
mvn clean deploy -P release --settings .travis/settings.xml
9+
mvn clean deploy -P release -P scala-2.12 --settings .travis/settings.xml

‎pom.xml

+67-14
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<modelVersion>4.0.0</modelVersion>
66

77
<groupId>com.amazon.deequ</groupId>
8-
<artifactId>deequ</artifactId>
8+
<artifactId>deequ${artifact.scala.version}</artifactId>
99
<version>1.1.0-SNAPSHOT</version>
1010

1111
<name>deequ</name>
@@ -59,8 +59,16 @@
5959
<maven.compiler.target>1.8</maven.compiler.target>
6060
<encoding>UTF-8</encoding>
6161
<scala.major.version>2.11</scala.major.version>
62-
<scala.version>${scala.major.version}.5</scala.version>
63-
<scala-maven-plugin.version>3.4.4</scala-maven-plugin.version>
62+
<scala.version>${scala.major.version}.10</scala.version>
63+
<artifact.scala.version></artifact.scala.version>
64+
<scala-maven-plugin.version>4.4.0</scala-maven-plugin.version>
65+
<!-- Spark -->
66+
67+
<spark.version>${spark-24.version}</spark.version>
68+
<spark-22.version>2.2.2</spark-22.version>
69+
<spark-23.version>2.3.2</spark-23.version>
70+
<spark-24.version>2.4.2</spark-24.version>
71+
<spark-30.version>3.0.0</spark-30.version>
6472
</properties>
6573

6674
<dependencies>
@@ -73,13 +81,13 @@
7381
<dependency>
7482
<groupId>org.apache.spark</groupId>
7583
<artifactId>spark-core_${scala.major.version}</artifactId>
76-
<version>2.3.3</version>
84+
<version>${spark.version}</version>
7785
</dependency>
7886

7987
<dependency>
8088
<groupId>org.apache.spark</groupId>
8189
<artifactId>spark-sql_${scala.major.version}</artifactId>
82-
<version>2.3.3</version>
90+
<version>${spark.version}</version>
8391
</dependency>
8492

8593
<dependency>
@@ -91,14 +99,14 @@
9199
<dependency>
92100
<groupId>org.scalatest</groupId>
93101
<artifactId>scalatest_${scala.major.version}</artifactId>
94-
<version>2.2.6</version>
102+
<version>3.1.2</version>
95103
<scope>test</scope>
96104
</dependency>
97105

98106
<dependency>
99107
<groupId>org.scalamock</groupId>
100-
<artifactId>scalamock-scalatest-support_${scala.major.version}</artifactId>
101-
<version>3.2.2</version>
108+
<artifactId>scalamock_${scala.major.version}</artifactId>
109+
<version>4.4.0</version>
102110
<scope>test</scope>
103111
</dependency>
104112

@@ -107,33 +115,33 @@
107115
<artifactId>scala-compiler</artifactId>
108116
<version>${scala.version}</version>
109117
<scope>test</scope>
110-
</dependency>
118+
</dependency>
111119

112120
<dependency>
113121
<groupId>org.mockito</groupId>
114122
<artifactId>mockito-core</artifactId>
115-
<version>2.21.0</version>
123+
<version>2.28.2</version>
116124
<scope>test</scope>
117125
</dependency>
118126

119127
<dependency>
120128
<groupId>org.openjdk.jmh</groupId>
121129
<artifactId>jmh-core</artifactId>
122-
<version>1.22</version>
130+
<version>1.23</version>
123131
<scope>test</scope>
124132
</dependency>
125133

126134
<dependency>
127135
<groupId>org.openjdk.jmh</groupId>
128136
<artifactId>jmh-generator-annprocess</artifactId>
129-
<version>1.22</version>
137+
<version>1.23</version>
130138
<scope>test</scope>
131139
</dependency>
132140

133141
<dependency>
134142
<groupId>org.apache.datasketches</groupId>
135143
<artifactId>datasketches-java</artifactId>
136-
<version>1.1.0-incubating</version>
144+
<version>1.3.0-incubating</version>
137145
<scope>test</scope>
138146
</dependency>
139147

@@ -226,6 +234,19 @@
226234
</executions>
227235
</plugin>
228236

237+
<!-- added for creating test jar to be ran on spark clusters for additional testing -->
238+
<plugin>
239+
<groupId>org.apache.maven.plugins</groupId>
240+
<artifactId>maven-jar-plugin</artifactId>
241+
<version>3.2.0</version>
242+
<executions>
243+
<execution>
244+
<goals>
245+
<goal>test-jar</goal>
246+
</goals>
247+
</execution>
248+
</executions>
249+
</plugin>
229250
<plugin>
230251
<groupId>org.apache.maven.plugins</groupId>
231252
<artifactId>maven-resources-plugin</artifactId>
@@ -377,6 +398,38 @@
377398
</plugins>
378399
</build>
379400
</profile>
401+
<profile>
402+
<id>scala-2.12</id>
403+
<properties>
404+
<scala.major.version>2.12</scala.major.version>
405+
<scala.version>${scala.major.version}.10</scala.version>
406+
<artifact.scala.version>_${scala.major.version}</artifact.scala.version>
407+
</properties>
408+
</profile>
409+
<profile>
410+
<id>spark-2.2</id>
411+
<properties>
412+
<spark.version>${spark-22.version}</spark.version>
413+
</properties>
414+
</profile>
415+
<profile>
416+
<id>spark-2.3</id>
417+
<properties>
418+
<spark.version>${spark-23.version}</spark.version>
419+
</properties>
420+
</profile>
421+
<profile>
422+
<id>spark-2.4</id>
423+
<properties>
424+
<spark.version>${spark-24.version}</spark.version>
425+
</properties>
426+
</profile>
427+
<profile>
428+
<id>spark-3.0</id>
429+
<properties>
430+
<spark.version>${spark-30.version}</spark.version>
431+
</properties>
432+
</profile>
380433
</profiles>
381434

382-
</project>
435+
</project>

‎src/main/scala/com/amazon/deequ/analyzers/applicability/Applicability.scala

+9-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.amazon.deequ.analyzers.applicability
1818

1919
import java.sql.Timestamp
20+
import java.util.Calendar
2021

2122
import com.amazon.deequ.analyzers.{Analyzer, State}
2223
import com.amazon.deequ.checks.Check
@@ -140,7 +141,14 @@ private[deequ] object Applicability {
140141
if (shouldBeNull(nullable)) {
141142
null
142143
} else {
143-
new Timestamp(Random.nextLong())
144+
val Low = 100
145+
val High = 1500
146+
val Result = Random.nextInt(High - Low) + Low
147+
val ResultSec = Random.nextInt(High - Low) + Low
148+
val calendar = Calendar.getInstance()
149+
calendar.add(Calendar.MINUTE, - Result)
150+
calendar.add(Calendar.SECOND, - ResultSec)
151+
new Timestamp(calendar.getTimeInMillis())
144152
}
145153
}
146154

‎src/main/scala/com/amazon/deequ/constraints/AnalysisBasedConstraint.scala

+23-20
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
/**
2-
* Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3-
*
4-
* Licensed under the Apache License, Version 2.0 (the "License"). You may not
5-
* use this file except in compliance with the License. A copy of the License
6-
* is located at
7-
*
8-
* http://aws.amazon.com/apache2.0/
9-
*
10-
* or in the "license" file accompanying this file. This file is distributed on
11-
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12-
* express or implied. See the License for the specific language governing
13-
* permissions and limitations under the License.
14-
*
15-
*/
2+
* Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not
5+
* use this file except in compliance with the License. A copy of the License
6+
* is located at
7+
*
8+
* http://aws.amazon.com/apache2.0/
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed on
11+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*
15+
*/
1616

1717
package com.amazon.deequ.constraints
1818

@@ -40,10 +40,10 @@ import scala.util.{Failure, Success, Try}
4040
*
4141
*/
4242
private[deequ] case class AnalysisBasedConstraint[S <: State[S], M, V](
43-
analyzer: Analyzer[S, Metric[M]],
44-
private[deequ] val assertion: V => Boolean,
45-
private[deequ] val valuePicker: Option[M => V] = None,
46-
private[deequ] val hint: Option[String] = None)
43+
analyzer: Analyzer[S, Metric[M]],
44+
private[deequ] val assertion: V => Boolean,
45+
private[deequ] val valuePicker: Option[M => V] = None,
46+
private[deequ] val hint: Option[String] = None)
4747
extends Constraint {
4848

4949
private[deequ] def calculateAndEvaluate(data: DataFrame) = {
@@ -52,8 +52,8 @@ private[deequ] case class AnalysisBasedConstraint[S <: State[S], M, V](
5252
}
5353

5454
override def evaluate(
55-
analysisResults: Map[Analyzer[_, Metric[_]], Metric[_]])
56-
: ConstraintResult = {
55+
analysisResults: Map[Analyzer[_, Metric[_]], Metric[_]])
56+
: ConstraintResult = {
5757

5858
val metric = analysisResults.get(analyzer).map(_.asInstanceOf[Metric[M]])
5959

@@ -110,6 +110,9 @@ private[deequ] case class AnalysisBasedConstraint[S <: State[S], M, V](
110110
case e: Exception => throw AnalysisBasedConstraint.ConstraintAssertionException(e.getMessage)
111111
}
112112

113+
// 'assertion' and 'valuePicker' are lambdas we have to represent them like '<function1>'
114+
override def toString: String =
115+
s"AnalysisBasedConstraint($analyzer,<function1>,${valuePicker.map(_ => "<function1>")},$hint)"
113116
}
114117

115118
private[deequ] object AnalysisBasedConstraint {

‎src/test/resources/EMRSparkShellTest.scala

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/**
2+
* Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not
5+
* use this file except in compliance with the License. A copy of the License
6+
* is located at
7+
*
8+
* http://aws.amazon.com/apache2.0/
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed on
11+
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*
15+
*/
16+
17+
/*
18+
For testing inside EMR or other flavors of spark cluster. Run commands after building git repo from source.
19+
Add additional test classes as needed
20+
scala 2.12
21+
spark-shell -i <local_git_repo>/src/test/resources/EMRSparkShellTest.txt \
22+
--packages org.scalatest:scalatest_2.12:3.1.2,org.scalamock:scalamock_2.12:4.4.0,org.scala-lang:scala-compiler:2.12.10,\
23+
org.mockito:mockito-core:2.28.2,org.openjdk.jmh:jmh-core:1.23,org.openjdk.jmh:jmh-generator-annprocess:1.23,org.apache.datasketches:datasketches-java:1.3.0-incubating \
24+
--jars <local_git_repo>/target/deequ_2.12-1.1.0-SNAPSHOT.jar,<local_git_repo>/target/deequ_2.12-1.1.0-SNAPSHOT-tests.jar
25+
26+
scala 2.11
27+
spark-shell -i <local_git_repo>/src/test/resources/EMRSparkShellTest.txt \
28+
--packages org.scalatest:scalatest_2.11:3.1.2,org.scalamock:scalamock_2.11:4.4.0,org.scala-lang:scala-compiler:2.11.10,\
29+
org.mockito:mockito-core:2.28.2,org.openjdk.jmh:jmh-core:1.23,org.openjdk.jmh:jmh-generator-annprocess:1.23,org.apache.datasketches:datasketches-java:1.3.0-incubating \
30+
--jars <local_git_repo>/target/deequ-1.1.0-SNAPSHOT.jar,<local_git_repo>/target/spark-deequ-testing/deequ-1.1.0-SNAPSHOT-tests.jar
31+
*/
32+
33+
import com.amazon.deequ.analyzers.{AnalysisTest, AnalyzerTests, IncrementalAnalysisTest}
34+
import com.amazon.deequ.analyzers.runners.{AnalysisRunnerTests, AnalyzerContextTest}
35+
import com.amazon.deequ.{VerificationResultTest, VerificationSuiteTest}
36+
37+
(new VerificationSuiteTest).execute()
38+
(new VerificationResultTest).execute()
39+
(new AnalysisRunnerTests).execute()
40+
(new AnalyzerContextTest).execute()
41+
(new AnalysisTest).execute()
42+
(new AnalyzerTests).execute()
43+
(new IncrementalAnalysisTest).execute()
44+
//Add additional test classes as needed

0 commit comments

Comments
 (0)