从csv文件名中提取日期并加载到-python pandas spark列中的hive表中

import pandas as pd
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf
from pyspark.sql.types import DateType

spark = SparkSession.builder.appName("CSV to Hive").getOrCreate()

df = pd.read_csv("file.csv")
df['Date'] = pd.to_datetime(df['Filename'].str.extract(r'(\d{4}-\d{2}-\d{2})'), format='%Y-%m-%d')

spark_df = spark.createDataFrame(df)

date_udf = udf(lambda x: pd.to_datetime(x).date(), DateType())
spark.udf.register("date_udf", date_udf)

spark_df.createOrReplaceTempView("temp_view")

spark.sql("INSERT INTO TABLE hive_table SELECT *, date_udf(Filename) AS Date FROM temp_view")