Блог

Я исследовал Google, но не нашел решения, поэтому разместил его здесь.

 val a_spark: SparkSession = SparkUtils.getSparkInstance("abc")

filepath : /Users/user1/Documents/input/demo.xml
 

использование вышеуказанных переменных в приведенном ниже методе

 def getDataFrame(a_spark: SparkSession, filePath: String): DataFrame = {
    a_spark.read
      .format("com.databricks.spark.xml")
      .option("rootTag", "PlaceList")
      .option("rowTag", "Place")
      .load(filePath) //error on this line
  }

Exception in thread "main" java.lang.ExceptionInInitializerError
    at Main$.delayedEndpoint$Main$1(Main.scala:8)
    at Main$delayedInit$body.apply(Main.scala:3)
    at scala.Function0.apply$mcV$sp(Function0.scala:39)
    at scala.Function0.apply$mcV$sp$(Function0.scala:39)
    at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:17)
    at scala.App.$anonfun$main$1$adapted(App.scala:80)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at scala.App.main(App.scala:80)
    at scala.App.main$(App.scala:78)
    at Main$.main(Main.scala:3)
    at Main.main(Main.scala)
Caused by: java.io.IOException: No FileSystem for scheme: file
    at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2586)
    at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2593)
    at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91)
    at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2632)
    at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2614)
    at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370)
    at org.apache.hadoop.fs.FileSystem.getLocal(FileSystem.java:341)
    at org.apache.spark.SparkContext.$anonfun$newAPIHadoopFile$2(SparkContext.scala:1151)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
    at org.apache.spark.SparkContext.withScope(SparkContext.scala:699)
    at org.apache.spark.SparkContext.newAPIHadoopFile(SparkContext.scala:1146)
    at com.databricks.spark.xml.util.XmlFile$.withCharset(XmlFile.scala:46)
    at com.databricks.spark.xml.DefaultSource.$anonfun$createRelation$1(DefaultSource.scala:71)
    at com.databricks.spark.xml.XmlRelation.$anonfun$schema$1(XmlRelation.scala:43)
    at scala.Option.getOrElse(Option.scala:189)
    at com.databricks.spark.xml.XmlRelation.<init>(XmlRelation.scala:42)
    at com.databricks.spark.xml.XmlRelation$.apply(XmlRelation.scala:29)
    at com.databricks.spark.xml.DefaultSource.createRelation(DefaultSource.scala:74)
    at com.databricks.spark.xml.DefaultSource.createRelation(DefaultSource.scala:52)
    at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:318)
    at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:223)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:211)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:178)
    at util.SparkUtils$.getDataFrame(SparkUtils.scala:26)
 

попытался добавить префикс file:// в начале пути к файлу, но после этого я также получаю ту же ошибку.

решением этого вопроса является добавление конфигураций файловой системы в sparksession.

 object IdentifyData {

  val m_spark: SparkSession = SparkUtils.getSparkInstance("name1")

  val hadoopConfig: Configuration = m_spark.sparkContext.hadoopConfiguration
  hadoopConfig.set("fs.hdfs.impl", classOf[org.apache.hadoop.hdfs.DistributedFileSystem].getName)
  hadoopConfig.set("fs.file.impl", classOf[org.apache.hadoop.fs.LocalFileSystem].getName)