Блог

У меня возникла проблема при попытке использовать zeppelin для подключения к моему локальному кластеру spark. Когда я пытаюсь получить sparkcontext, он жалуется SERVICE_DOMAIN=null. Я развернул zeppelin и spark на одном компьютере и настроил spark.master и SPARK_HOME.

spark.master spark://192.168.1.5:7077 SPARK_HOME /opt/spark

  INFO [2020-12-16 14:43:36,369] ({qtp597255128-185} InterpreterSetting.java[createInterpreters]:844) - Interpreter org.apache.zeppelin.spark.SparkInterpreter created for user: anonymous, sessionId: shared_session
 INFO [2020-12-16 14:43:36,369] ({qtp597255128-185} InterpreterSetting.java[createInterpreters]:844) - Interpreter org.apache.zeppelin.spark.SparkSqlInterpreter created for user: anonymous, sessionId: shared_session
 INFO [2020-12-16 14:43:36,370] ({qtp597255128-185} InterpreterSetting.java[createInterpreters]:844) - Interpreter org.apache.zeppelin.spark.PySparkInterpreter created for user: anonymous, sessionId: shared_session
 INFO [2020-12-16 14:43:36,370] ({qtp597255128-185} InterpreterSetting.java[createInterpreters]:844) - Interpreter org.apache.zeppelin.spark.IPySparkInterpreter created for user: anonymous, sessionId: shared_session
 INFO [2020-12-16 14:43:36,370] ({qtp597255128-185} InterpreterSetting.java[createInterpreters]:844) - Interpreter org.apache.zeppelin.spark.SparkRInterpreter created for user: anonymous, sessionId: shared_session
 INFO [2020-12-16 14:43:36,371] ({qtp597255128-185} InterpreterSetting.java[createInterpreters]:844) - Interpreter org.apache.zeppelin.spark.SparkIRInterpreter created for user: anonymous, sessionId: shared_session
 INFO [2020-12-16 14:43:36,371] ({qtp597255128-185} InterpreterSetting.java[createInterpreters]:844) - Interpreter org.apache.zeppelin.spark.SparkShinyInterpreter created for user: anonymous, sessionId: shared_session
 INFO [2020-12-16 14:43:36,372] ({qtp597255128-185} InterpreterSetting.java[createInterpreters]:844) - Interpreter org.apache.zeppelin.spark.KotlinSparkInterpreter created for user: anonymous, sessionId: shared_session
 INFO [2020-12-16 14:43:36,372] ({qtp597255128-185} ManagedInterpreterGroup.java[getOrCreateSession]:179) - Create Session: shared_session in InterpreterGroup: spark-shared_process for user: anonymous
 INFO [2020-12-16 14:43:39,312] ({qtp597255128-189} NotebookService.java[runParagraph]:312) - Start to run paragraph: paragraph_1608089505700_614894903 of note: 2FU4YW6UB
 INFO [2020-12-16 14:43:39,312] ({qtp597255128-189} VFSNotebookRepo.java[save]:145) - Saving note 2FU4YW6UB to test_2FU4YW6UB.zpln
 INFO [2020-12-16 14:43:39,319] ({SchedulerFactory2} AbstractScheduler.java[runJob]:125) - Job paragraph_1608089505700_614894903 started by scheduler RemoteInterpreter-spark-shared_process-shared_session
 INFO [2020-12-16 14:43:39,321] ({SchedulerFactory2} Paragraph.java[jobRun]:388) - Run paragraph [paragraph_id: paragraph_1608089505700_614894903, interpreter: org.apache.zeppelin.spark.SparkInterpreter, note_id: 2FU4YW6UB, user: anonymous]
 INFO [2020-12-16 14:43:39,321] ({SchedulerFactory2} ManagedInterpreterGroup.java[getOrCreateInterpreterProcess]:64) - Create InterpreterProcess for InterpreterGroup: spark-shared_process
 INFO [2020-12-16 14:43:39,322] ({SchedulerFactory2} K8sStandardInterpreterLauncher.java[launchDirectly]:136) - Launching Interpreter: spark
 WARN [2020-12-16 14:43:39,327] ({SchedulerFactory2} NotebookServer.java[onStatusChange]:1904) - Job paragraph_1608089505700_614894903 is finished, status: ERROR, exception: null, result: %text java.lang.NullPointerException: null value in entry: SERVICE_DOMAIN=null
        at com.google.common.collect.CollectPreconditions.checkEntryNotNull(CollectPreconditions.java:34)
        at com.google.common.collect.ImmutableMap.entryOf(ImmutableMap.java:175)
        at com.google.common.collect.ImmutableMap.of(ImmutableMap.java:141)
        at org.apache.zeppelin.interpreter.launcher.K8sRemoteInterpreterProcess.sparkUiWebUrlFromTemplate(K8sRemoteInterpreterProcess.java:330)
        at org.apache.zeppelin.interpreter.launcher.K8sRemoteInterpreterProcess.getTemplateBindings(K8sRemoteInterpreterProcess.java:299)
        at org.apache.zeppelin.interpreter.launcher.K8sRemoteInterpreterProcess.start(K8sRemoteInterpreterProcess.java:125)
        at org.apache.zeppelin.interpreter.ManagedInterpreterGroup.getOrCreateInterpreterProcess(ManagedInterpreterGroup.java:67)
        at org.apache.zeppelin.interpreter.remote.RemoteInterpreter.getOrCreateInterpreterProcess(RemoteInterpreter.java:110)
        at org.apache.zeppelin.interpreter.remote.RemoteInterpreter.internal_create(RemoteInterpreter.java:160)
        at org.apache.zeppelin.interpreter.remote.RemoteInterpreter.open(RemoteInterpreter.java:131)
        at org.apache.zeppelin.interpreter.remote.RemoteInterpreter.getFormType(RemoteInterpreter.java:281)
        at org.apache.zeppelin.notebook.Paragraph.jobRun(Paragraph.java:412)
        at org.apache.zeppelin.notebook.Paragraph.jobRun(Paragraph.java:72)
        at org.apache.zeppelin.scheduler.Job.run(Job.java:172)
        at org.apache.zeppelin.scheduler.AbstractScheduler.runJob(AbstractScheduler.java:130)
        at org.apache.zeppelin.scheduler.RemoteScheduler$JobRunner.run(RemoteScheduler.java:180)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748) ```