联合两个Spark数据帧并添加新列以标识最新日期

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, when
from pyspark.sql.types import StringType

spark = SparkSession.builder.getOrCreate()

df1 = spark.createDataFrame([(1, '2022-01-01'), (2, '2022-02-01')], ['id', 'date'])
df2 = spark.createDataFrame([(3, '2022-03-01'), (4, '2022-04-01')], ['id', 'date'])

union_df = df1.union(df2)

latest_date_df = union_df.groupBy('id').agg({'date': 'max'}).withColumnRenamed('max(date)', 'latest_date')

final_df = union_df.join(latest_date_df, on='id', how='left')

final_df = final_df.withColumn('is_latest_date', when(col('date') == col('latest_date'), 'Yes').otherwise('No'))

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, when
from pyspark.sql.types import StringType

spark = SparkSession.builder.getOrCreate()

df1 = spark.createDataFrame([(1, '2022-01-01'), (2, '2022-02-01')], ['id', 'date'])
df2 = spark.createDataFrame([(3, '2022-03-01'), (4, '2022-04-01')], ['id', 'date'])

union_df = df1.union(df2)

latest_date_df = union_df.groupBy('id').agg({'date': 'max'}).withColumnRenamed('max(date)', 'latest_date')

final_df = union_df.join(latest_date_df, on='id', how='left')

final_df = final_df.withColumn('is_latest_date', when(col('date') == col('latest_date'), 'Yes').otherwise('No'))

final_df.show()