使用pyspark解析Spark 3数据帧中的多行嵌套json

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, from_json, explode
from pyspark.sql.types import StructType

spark = SparkSession.builder \
    .appName("Nested JSON Parsing") \
    .getOrCreate()

schema = StructType() \
    .add("id", "integer") \
    .add("name", "string") \
    .add("address", StructType()
         .add("street", "string")
         .add("city", "string")
         .add("state", "string"))

dataframe = spark.read \
    .option("multiLine", True) \
    .json("path/to/json/file.json", schema=schema)

parsed_df = dataframe \
    .select("id", "name", 
            col("address.street").alias("street"), 
            col("address.city").alias("city"), 
            col("address.state").alias("state"))