Блог

Я работаю над разработкой многомодульного проекта в SBT. Идея состоит в том, чтобы создать два подпроекта в SBT, причем один подпроект имеет зависимость Apache spark 2.3, а другой-зависимость Apache Spark 2.4. Каждый подпроект должен быть способен создавать отдельные банки. Я вставляю файл build.sbt. В то время как я пытаюсь запустить сборку sbt в каждом из этих подпроектов. Я получаю следующую ошибку.Я попытался удалить папку кэша в .ivy2 и попытался воссоздать структуру проекта.
Сообщение об ошибке(включало только соответствующие журналы)

 [IJ]> multi2/assembly

[info] Including from cache: jackson-xc-1.9.13.jar
[info] Including from cache: minlog-1.3.0.jar
[warn] Merging 'META-INFASL2.0' with strategy 'discard'
[warn] Merging 'META-INFDEPENDENCIES' with strategy 'discard'

[error] C:Usersuser_home.ivy2cacheorg.apache.sparkspark-core_2.11jarsspark-core_2.11-2.4.3.jar:org/apache/spark/unused/UnusedStubClass.class

[error] C:Usersuser_home.ivy2cacheorg.apache.sparkspark-launcher_2.11jarsspark-launcher_2.11-2.4.3.jar:org/apache/spark/unused/UnusedStubClass.class

[error] C:Usersuser_home.ivy2cacheorg.apache.sparkspark-tags_2.11jarsspark-tags_2.11-2.4.3.jar:org/apache/spark/unused/UnusedStubClass.class

[error] C:Usersuser_home.ivy2cacheorg.spark-project.sparkunusedjarsunused-1.0.0.jar:org/apache/spark/unused/UnusedStubClass.class
 

Точно так же существуют и другие классы, которые вызывают ошибки, как указано выше..
Пожалуйста, предложите и дайте мне знать для получения любой дополнительной информации.

 name := "testrepo"
        organization in ThisBuild := "com.yt"
        scalaVersion in ThisBuild := "2.11.8"
        scalaBinaryVersion := "2.11"
        
        lazy val common = (project in file("common"))
          .settings(
            name := "common",
            commonSettings,
            dependencyOverrides  = "com.fasterxml.jackson.core" % "jackson-core" % "2.8.7",
            dependencyOverrides  = "com.fasterxml.jackson.core" % "jackson-databind" % "2.8.7",
            dependencyOverrides  = "com.fasterxml.jackson.module" %% "jackson-module-scala" % 
            dependencyOverrides  = "io.netty" % "netty" % "3.9.9.Final",
            dependencyOverrides  = "commons-net" % "commons-net" % "2.2",
            dependencyOverrides  = "com.google.guava" % "guava" % "11.0.2",
            dependencyOverrides  =  "com.google.code.findbugs" %  "jsr305" % "1.3.9",
            libraryDependencies   = commonDependencies
        
          )
          .disablePlugins(AssemblyPlugin)
        
        
        lazy val multi1 = (project in file("multi1"))
          .settings(
            name := "multi1",
            commonSettings,
            assemblySettings,
            libraryDependencies   = commonDependencies    Seq(
              dependencies.spark23
        
            )
          )
          .dependsOn(
            common
          )
        
        lazy val multi2 = (project in file("multi2"))
          .settings(
            name := "multi2",
            commonSettings,
            assemblySettings,
            libraryDependencies   = commonDependencies    Seq(
              dependencies.spark24
            )
          )
          .dependsOn(
            common
          )
        
         /*val overrides = Seq("com.fasterxml.jackson.core" % "jackson-core" % "2.8.7",
          "com.fasterxml.jackson.core" % "jackson-databind" % "2.8.7",
          "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.8.7") */
        // DEPENDENCIES
        
        lazy val dependencies =
          new {
            val hivejdbc="org.apache.hive" % "hive-jdbc" % "0.13.0" % "provided"
            val spark23V = "2.3.0.cloudera3"
            val spark24V="2.4.3"
            val spark23="org.apache.spark" %% "spark-core" % spark23V
            val spark24="org.apache.spark" %% "spark-core" % spark24V
          }
        
        lazy val commonDependencies = Seq(
          dependencies.hivejdbc
        )
        
        lazy val compilerOptions = Seq(
          "-unchecked",
          "-feature",
          "-language:existentials",
          "-language:higherKinds",
          "-language:implicitConversions",
          "-language:postfixOps",
          "-deprecation",
          "-encoding",
          "utf8"
        )
        
        lazy val commonSettings = Seq(
          scalacOptions   = compilerOptions,
          resolvers   = Seq(
            "Local Maven Repository" at "file://"   Path.userHome.absolutePath   "/.m2/repository",
            Resolver.sonatypeRepo("releases"),
            Resolver.sonatypeRepo("snapshots")
        
          )
        )
        
        lazy val assemblySettings = Seq(
          assemblyJarName in assembly := name.value   ".jar",
          assemblyMergeStrategy in assembly := {
            case PathList("org.apache.spark", "spark-core_2.11", xs @ _*) => MergeStrategy.last
            case PathList("META-INF", xs @ _*) => MergeStrategy.discard
            case "application.conf"            => MergeStrategy.concat
            case x =>
              val oldStrategy = (assemblyMergeStrategy in assembly).value
              oldStrategy(x)
          }
        )