Блог

Я пытаюсь упаковать файл jar. Я получаю ошибку ниже:

 [error] (*:packageOptions) Please add any Spark dependencies by supplying the sparkVersion and 
sparkComponents. Please remove: org.apache.spark:spark-core:2.3.0.cloudera3
 

Build.sbt выглядит следующим образом

  lazy val root = (project in file(".")).
     settings(
       name := "testrepo_9010",
       version := "1.0",
       scalaVersion := "2.11.8",
       mainClass in Compile := Some("com.example.test.Sparktesting")
     )


    //name:= "testrepo_9010"
    //
    //version:="1.0"
    //
    //scalaVersion:= "2.11.8"
    
    
    val sparkVersiontest = "2.3.0.cloudera3"
    
    assemblyJarName in assembly := s"${name.value}_${scalaBinaryVersion.value}-${version.value}.jar"
    
    artifact in(Compile, assembly) := {
      val art = (artifact in(Compile, assembly)).value
      art.copy(`classifier` = Some("assembly"))
    }
    addArtifact(artifact in(Compile, assembly), assembly)
    
    
    dependencyOverrides  = "com.fasterxml.jackson.core" % "jackson-core" % "2.8.7"
    dependencyOverrides  = "com.fasterxml.jackson.core" % "jackson-databind" % "2.8.7"
    dependencyOverrides  = "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.8.7"
    
    libraryDependencies   = Seq(
      "org.apache.hive" % "hive-jdbc" % "0.13.0" % "provided",
      "org.apache.hive" % "hive-exec" % "0.13.0" % "provided",
      "org.apache.hadoop" % "hadoop-common" % "2.6.0-cdh5.8.4" % "provided",
      "org.apache.thrift" % "libthrift" % "0.9.0" % "provided",
      "org.apache.thrift" % "libfb303" % "0.9.0" % "provided",
      "commons-logging" % "commons-logging" % "1.1.3" % "provided",
      "org.apache.hadoop" % "hadoop-client" % "2.6.0" % "provided",
      "com.oracle" %"ojdbc6"%"11.2.0.3" %"provided"
    
    )
    
    libraryDependencies   = Seq(
      "org.apache.spark" %% s"spark-core" % sparkVersiontest ,
      "org.apache.spark" %% s"spark-sql" % sparkVersiontest ,
      "org.apache.spark" %% s"spark-hive" % sparkVersiontest
      
    )
    
     assemblyMergeStrategy in assembly := {
       case PathList("META-INF", xs@_*) => MergeStrategy.discard
       case x => MergeStrategy.first
     }