Блог

привет, я изучаю spark с помощью scala. я запускаю код, чтобы узнать, сколько слов в файле и частота каждого слова. но когда я запускаю код, возникает эта ошибка.

  java.lang.NoSuchMethodError: scala.Predef$.refArrayOps([Ljava/lang/Object;)Lscala/collection/mutable/ArrayOps;
        at example.word_count$anonfun$2.apply(word_count.scala:19)
        at example.word_count$anonfun$2.apply(word_count.scala:19)
        at scala.collection.Iterator$anon$11.nextCur(Iterator.scala:484)
        at scala.collection.Iterator$anon$11.hasNext(Iterator.scala:490)
        at scala.collection.Iterator$anon$10.hasNext(Iterator.scala:458)
        at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:192)
        at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:62)
        at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)
        at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:99)
        at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:52)
        at org.apache.spark.scheduler.Task.run(Task.scala:131)
        at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:497)
        at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1439)
        at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:500)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
 

это build.sbt

 val sparkVersion = "3.1.2"
ThisBuild / scalaVersion     := "2.11.12"
ThisBuild / version          := "0.1.0-SNAPSHOT"
ThisBuild / organization     := "com.example"
ThisBuild / organizationName := "example"

lazy val root = (project in file("."))
  .settings(
    name := "word_count",
  )


libraryDependencies   = Seq(
  "org.apache.spark" % "spark-core_2.11" % "2.4.5",
  
  "org.apache.spark" %  "spark-mllib_2.11" % "2.4.0",
  
  "org.apache.spark" % "spark-streaming_2.11" % "2.1.1",

  "org.apache.spark" % "spark-sql_2.11" % "2.1.0",
  
  "org.scalactic" %% "scalactic" % "3.2.10",

  "org.scalatest" %% "scalatest" % "3.2.10" % "test"
)

assemblyMergeStrategy in assembly := {
 case PathList("META-INF", xs @ _*) => MergeStrategy.discard
 case x => MergeStrategy.first
}
 

как это исправить, любая помощь будет признательна