使用pyspark验证同一列不同行中的数据

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, lag
from pyspark.sql.window import Window

spark = SparkSession.builder.appName("DataValidation").getOrCreate()

df = spark.read.csv("path_to_file.csv", header=True, inferSchema=True)

windowSpec = Window.partitionBy().orderBy("column_name")
df = df.withColumn("previous_value", lag(col("column_name")).over(windowSpec))

df = df.withColumn("validation", col("column_name") == col("previous_value"))

df.show()