holdenk / holdenk/spark-testing-base
Expected instance not static method org.scalatest.Assertions in DataFrame and DataSet SuitBase
- Dominant language
- Scala
- Stars
- 1.6k
- Forks
- 357
- PR merge metrics
- No merged PRs in 30d
Description
I'm writing the samples from the wiki of this repo and got some very perplexed exceptions which I can't understand
My dependencies are:
```
org.scalatest
scalatest_2.12
3.2.9
test
com.holdenkarau
spark-testing-base_2.12
3.1.2_1.1.0
test
```
and my spark version is 3.1.2, scala version is 2.12.12
My test code is mostly like that in the wiki page:
```
class test extends AnyFunSuite with DataFrameSuiteBase {
test("simple test") {
// val list = List(1, 2, 3, 4)
// val rdd = spark.sparkContext.parallelize(list)
// assert(rdd.count === list.length)
import spark.implicits._
val input1 = sc.parallelize(List[(Int, Double)]((1, 1.1), (2, 2.2), (3, 3.3))).toDF
val input2 = sc.parallelize(List[(Int, Double)]((1, 1.2), (2, 2.3), (3, 3.4))).toDF
input1.printSchema
input2.printSchema
assertDataFrameEquals(input1, input2) // equal
intercept[org.scalatest.exceptions.TestFailedException] {
assertDataFrameApproximateEquals(input1, input2, 0.05) // not equal
}
}
}
class TestDataSet extends AnyFunSuite with DatasetSuiteBase {
test("simple test") {
val sqlCtx = sqlContext
import sqlCtx.implicits._
val input1 = sc.parallelize(List(1, 2, 3)).toDS
assertDatasetEquals(input1, input1) // equal
val input2 = sc.parallelize(List(4, 5, 6)).toDS
intercept[org.scalatest.exceptions.TestFailedException] {
assertDatasetEquals(input1, input2) // not equal
}
}
}
```
But both test classes all end up with the same error:
```
An exception or error caused a run to abort: Expected instance not static method org.scalatest.Assertions.assertionsHelper()Lorg/scalatest/Assertions$AssertionsHelper;
java.lang.IncompatibleClassChangeError: Expected instance not static method org.scalatest.Assertions.assertionsHelper()Lorg/scalatest/Assertions$AssertionsHelper;
at com.holdenkarau.spark.testing.TestSuite.assert(TestSuite.scala:19)
at com.holdenkarau.spark.testing.TestSuite.assert$(TestSuite.scala:18)
at spark_testing_base.TestDataSet.assert(SampleTest.scala:47)
at com.holdenkarau.spark.testing.DatasetSuiteBaseLike.assertDatasetEquals(DatasetSuiteBase.scala:51)
at com.holdenkarau.spark.testing.DatasetSuiteBaseLike.assertDatasetEquals$(DatasetSuiteBase.scala:46)
at spark_testing_base.TestDataSet.assertDatasetEquals(SampleTest.scala:47)
at spark_testing_base.TestDataSet.$anonfun$new$5(SampleTest.scala:53)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.funsuite.AnyFunSuiteLike$$anon$1.apply(AnyFunSuiteLike.scala:190)
at org.scalatest.TestSuite.withFixture(TestSuite.scala:196)
at org.scalatest.TestSuite.withFixture$(TestSuite.scala:195)
at org.scalatest.funsuite.AnyFunSuite.withFixture(AnyFunSuite.scala:1563)
at org.scalatest.funsuite.AnyFunSuiteLike.invokeWithFixture$1(AnyFunSuiteLike.scala:188)
at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTest$1(AnyFunSuiteLike.scala:200)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
at org.scalatest.funsuite.AnyFunSuiteLike.runTest(AnyFunSuiteLike.scala:200)
at org.scalatest.funsuite.AnyFunSuiteLike.runTest$(AnyFunSuiteLike.scala:182)
at org.scalatest.funsuite.AnyFunSuite.runTest(AnyFunSuite.scala:1563)
at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTests$1(AnyFunSuiteLike.scala:233)
at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:413)
at scala.collection.immutable.List.foreach(List.scala:431)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:396)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:475)
at org.scalatest.funsuite.AnyFunSuiteLike.runTests(AnyFunSuiteLike.scala:233)
at org.scalatest.funsuite.AnyFunSuiteLike.runTests$(AnyFunSuiteLike.scala:232)
at org.scalatest.funsuite.AnyFunSuite.runTests(AnyFunSuite.scala:1563)
at org.scalatest.Suite.run(Suite.scala:1112)
at org.scalatest.Suite.run$(Suite.scala:1094)
at org.scalatest.funsuite.AnyFunSuite.org$scalatest$funsuite$AnyFunSuiteLike$$super$run(AnyFunSuite.scala:1563)
at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$run$1(AnyFunSuiteLike.scala:237)
at org.scalatest.SuperEngine.runImpl(Engine.scala:535)
at org.scalatest.funsuite.AnyFunSuiteLike.run(AnyFunSuiteLike.scala:237)
at org.scalatest.funsuite.AnyFunSuiteLike.run$(AnyFunSuiteLike.scala:236)
at spark_testing_base.TestDataSet.org$scalatest$BeforeAndAfterAll$$super$run(SampleTest.scala:47)
at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at spark_testing_base.TestDataSet.run(SampleTest.scala:47)
at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13(Runner.scala:1320)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13$adapted(Runner.scala:1314)
at scala.collection.immutable.List.foreach(List.scala:431)
at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1314)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:993)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:971)
at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1480)
at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:971)
at org.scalatest.tools.Runner$.run(Runner.scala:798)
at org.scalatest.tools.Runner.run(Runner.scala)
at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2or3(ScalaTestRunner.java:38)
at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:25)
```
How to fix this?
Contributor guide
Research direction
Reproduce the wiki sample with Spark 3.1.2, Scala 2.12.12, ScalaTest 3.2.9, and spark-testing-base 3.1.2_1.1.0. Start at TestSuite.scala:19 and DatasetSuiteBase.scala:51, then check the dependency compatibility around the reported Assertions method error. Done means both DataFrameSuiteBase and DatasetSuiteBase examples run without the IncompatibleClassChangeError.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- scala
- Domain
- testing
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Mostly clear
- Newbie friendliness
- 38/100