SparkSession sparkSession = SparkSession.builder().
master(master).
appName(Math.random()+"").
config("spark.some.config.option", "some-value").
// config("spark.driver.host","localhost").
getOrCreate();
错误信息如下:
11:43:02.875 [main] DEBUG org.spark_project.jetty.util.Jetty -
java.lang.NumberFormatException: For input string: "unknown"
at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
at java.lang.Long.parseLong(Long.java:589)
at java.lang.Long.valueOf(Long.java:803)
at org.spark_project.jetty.util.Jetty.formatTimestamp(Jetty.java:89)
at org.spark_project.jetty.util.Jetty.<clinit>(Jetty.java:61)
at org.spark_project.jetty.server.Server.getVersion(Server.java:159)
at org.spark_project.jetty.server.handler.ContextHandler.<clinit>(ContextHandler.java:128)
at org.apache.spark.ui.JettyUtils$.createServletHandler(JettyUtils.scala:143)
at org.apache.spark.ui.JettyUtils$.createServletHandler(JettyUtils.scala:130)
at org.apache.spark.ui.WebUI.attachPage(WebUI.scala:83)
at org.apache.spark.ui.WebUI$$anonfun$attachTab$1.apply(WebUI.scala:65)
at org.apache.spark.ui.WebUI$$anonfun$attachTab$1.apply(WebUI.scala:65)
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at org.apache.spark.ui.WebUI.attachTab(WebUI.scala:65)
at org.apache.spark.ui.SparkUI.initialize(SparkUI.scala:62)
at org.apache.spark.ui.SparkUI.<init>(SparkUI.scala:80)
at org.apache.spark.ui.SparkUI$.create(SparkUI.scala:175)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:444)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2520)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:935)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:926)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
Exception in thread "main" java.lang.IllegalArgumentException: Illegal pattern component: XXX
at org.apache.commons.lang3.time.FastDatePrinter.parsePattern(FastDatePrinter.java:282)
at org.apache.commons.lang3.time.FastDatePrinter.init(FastDatePrinter.java:149)
at org.apache.commons.lang3.time.FastDatePrinter.<init>(FastDatePrinter.java:142)
at org.apache.commons.lang3.time.FastDateFormat.<init>(FastDateFormat.java:384)
at org.apache.commons.lang3.time.FastDateFormat.<init>(FastDateFormat.java:369)
at org.apache.commons.lang3.time.FastDateFormat$1.createInstance(FastDateFormat.java:91)
at org.apache.commons.lang3.time.FastDateFormat$1.createInstance(FastDateFormat.java:88)
at org.apache.commons.lang3.time.FormatCache.getInstance(FormatCache.java:82)
at org.apache.commons.lang3.time.FastDateFormat.getInstance(FastDateFormat.java:165)
at org.apache.spark.sql.execution.datasources.csv.CSVOptions.<init>(CSVOptions.scala:139)
at org.apache.spark.sql.execution.datasources.csv.CSVOptions.<init>(CSVOptions.scala:41)
at org.apache.spark.sql.execution.datasources.csv.CSVFileFormat.inferSchema(CSVFileFormat.scala:58)
at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$6.apply(DataSource.scala:180)
at org.apache.spark.sql.execution.datasources.DataSource$$anonfun$6.apply(DataSource.scala:180)
at scala.Option.orElse(Option.scala:289)
at org.apache.spark.sql.execution.datasources.DataSource.getOrInferFileFormatSchema(DataSource.scala:179)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:373)
at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:223)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:211)
at org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:617)
at org.apache.spark.sql.DataFrameReader.csv(DataFrameReader.scala:468)
依赖如下:
<properties>
<java.version>1.8</java.version>
<spark.version>2.4.0</spark.version>
<hadoop.version>2.7.7</hadoop.version>
<spark.scala.version>2.11</spark.scala.version>
</properties>
相似问题