如何在Spark Scala中进行窗口划分并仅提取每个组的唯一值

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.functions._

object UniqueValuesInWindow {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .appName("Unique Values in Window")
      .master("local[*]")
      .getOrCreate()

    import spark.implicits._

    // 创建示例数据
    val data = Seq(
      (1, "A", 10),
      (2, "A", 20),
      (3, "A", 10), // 重复值
      (4, "B", 30),
      (5, "B", 40),
      (6, "B", 30)  // 重复值
    )

    val df = data.toDF("id", "category", "value")

    // 定义窗口规范
    val windowSpec = Window.partitionBy("category").orderBy("value")

    // 使用row_number()来标记重复值
    val dfWithRowNumber = df.withColumn("row_num", row_number().over(windowSpec))

    // 过滤掉重复值，只保留每组的第一行
    val uniqueValuesDF = dfWithRowNumber.filter($"row_num" === 1).drop("row_num")

    uniqueValuesDF.show()
  }
}