holdenk / holdenk/spark-testing-base

Expected instance not static method org.scalatest.Assertions in DataFrame and DataSet SuitBase

Open
#342 1 comment 0 reactions 0 assignees View on GitHub
Dominant language
Scala
Stars
1.6k
Forks
357
PR merge metrics
No merged PRs in 30d

Description

I'm writing the samples from the wiki of this repo and got some very perplexed exceptions which I can't understand

My dependencies are:
```

org.scalatest
scalatest_2.12
3.2.9
test


com.holdenkarau
spark-testing-base_2.12
3.1.2_1.1.0
test

```
and my spark version is 3.1.2, scala version is 2.12.12

My test code is mostly like that in the wiki page:
```
class test extends AnyFunSuite with DataFrameSuiteBase {
test("simple test") {
// val list = List(1, 2, 3, 4)
// val rdd = spark.sparkContext.parallelize(list)
// assert(rdd.count === list.length)
import spark.implicits._

val input1 = sc.parallelize(List[(Int, Double)]((1, 1.1), (2, 2.2), (3, 3.3))).toDF
val input2 = sc.parallelize(List[(Int, Double)]((1, 1.2), (2, 2.3), (3, 3.4))).toDF
input1.printSchema
input2.printSchema
assertDataFrameEquals(input1, input2) // equal

intercept[org.scalatest.exceptions.TestFailedException] {
assertDataFrameApproximateEquals(input1, input2, 0.05) // not equal
}
}
}

class TestDataSet extends AnyFunSuite with DatasetSuiteBase {
test("simple test") {
val sqlCtx = sqlContext
import sqlCtx.implicits._

val input1 = sc.parallelize(List(1, 2, 3)).toDS
assertDatasetEquals(input1, input1) // equal

val input2 = sc.parallelize(List(4, 5, 6)).toDS
intercept[org.scalatest.exceptions.TestFailedException] {
assertDatasetEquals(input1, input2) // not equal
}
}
}
```

But both test classes all end up with the same error:
```
An exception or error caused a run to abort: Expected instance not static method org.scalatest.Assertions.assertionsHelper()Lorg/scalatest/Assertions$AssertionsHelper;
java.lang.IncompatibleClassChangeError: Expected instance not static method org.scalatest.Assertions.assertionsHelper()Lorg/scalatest/Assertions$AssertionsHelper;
at com.holdenkarau.spark.testing.TestSuite.assert(TestSuite.scala:19)
at com.holdenkarau.spark.testing.TestSuite.assert$(TestSuite.scala:18)
at spark_testing_base.TestDataSet.assert(SampleTest.scala:47)
at com.holdenkarau.spark.testing.DatasetSuiteBaseLike.assertDatasetEquals(DatasetSuiteBase.scala:51)
at com.holdenkarau.spark.testing.DatasetSuiteBaseLike.assertDatasetEquals$(DatasetSuiteBase.scala:46)
at spark_testing_base.TestDataSet.assertDatasetEquals(SampleTest.scala:47)
at spark_testing_base.TestDataSet.$anonfun$new$5(SampleTest.scala:53)
at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
at org.scalatest.Transformer.apply(Transformer.scala:22)
at org.scalatest.Transformer.apply(Transformer.scala:20)
at org.scalatest.funsuite.AnyFunSuiteLike$$anon$1.apply(AnyFunSuiteLike.scala:190)
at org.scalatest.TestSuite.withFixture(TestSuite.scala:196)
at org.scalatest.TestSuite.withFixture$(TestSuite.scala:195)
at org.scalatest.funsuite.AnyFunSuite.withFixture(AnyFunSuite.scala:1563)
at org.scalatest.funsuite.AnyFunSuiteLike.invokeWithFixture$1(AnyFunSuiteLike.scala:188)
at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTest$1(AnyFunSuiteLike.scala:200)
at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
at org.scalatest.funsuite.AnyFunSuiteLike.runTest(AnyFunSuiteLike.scala:200)
at org.scalatest.funsuite.AnyFunSuiteLike.runTest$(AnyFunSuiteLike.scala:182)
at org.scalatest.funsuite.AnyFunSuite.runTest(AnyFunSuite.scala:1563)
at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTests$1(AnyFunSuiteLike.scala:233)
at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:413)
at scala.collection.immutable.List.foreach(List.scala:431)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:396)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:475)
at org.scalatest.funsuite.AnyFunSuiteLike.runTests(AnyFunSuiteLike.scala:233)
at org.scalatest.funsuite.AnyFunSuiteLike.runTests$(AnyFunSuiteLike.scala:232)
at org.scalatest.funsuite.AnyFunSuite.runTests(AnyFunSuite.scala:1563)
at org.scalatest.Suite.run(Suite.scala:1112)
at org.scalatest.Suite.run$(Suite.scala:1094)
at org.scalatest.funsuite.AnyFunSuite.org$scalatest$funsuite$AnyFunSuiteLike$$super$run(AnyFunSuite.scala:1563)
at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$run$1(AnyFunSuiteLike.scala:237)
at org.scalatest.SuperEngine.runImpl(Engine.scala:535)
at org.scalatest.funsuite.AnyFunSuiteLike.run(AnyFunSuiteLike.scala:237)
at org.scalatest.funsuite.AnyFunSuiteLike.run$(AnyFunSuiteLike.scala:236)
at spark_testing_base.TestDataSet.org$scalatest$BeforeAndAfterAll$$super$run(SampleTest.scala:47)
at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at spark_testing_base.TestDataSet.run(SampleTest.scala:47)
at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13(Runner.scala:1320)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13$adapted(Runner.scala:1314)
at scala.collection.immutable.List.foreach(List.scala:431)
at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1314)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:993)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:971)
at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1480)
at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:971)
at org.scalatest.tools.Runner$.run(Runner.scala:798)
at org.scalatest.tools.Runner.run(Runner.scala)
at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2or3(ScalaTestRunner.java:38)
at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:25)

```

How to fix this?

Contributor guide

Open the contributing guide

Research direction

Reproduce the wiki sample with Spark 3.1.2, Scala 2.12.12, ScalaTest 3.2.9, and spark-testing-base 3.1.2_1.1.0. Start at TestSuite.scala:19 and DatasetSuiteBase.scala:51, then check the dependency compatibility around the reported Assertions method error. Done means both DataFrameSuiteBase and DatasetSuiteBase examples run without the IncompatibleClassChangeError.

Written by the indexing model from the issue text.

Assessment

Tech stack
scala
Domain
testing
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
38/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.