在Pyspark中高效地从多个列中独立查找top-N值

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, desc

spark = SparkSession.builder.appName("TopNValues").getOrCreate()

data = spark.read.csv("data.csv", header=True, inferSchema=True)

columns = ["col1", "col2", "col3"]

topN = 5  # 要查找的top-N值的数量
results = []
for column in columns:
    topN_values = data.select(column).distinct().orderBy(col(column).desc()).limit(topN)
    results.append(topN_values)

for i, column in enumerate(columns):
    print(f"Top {topN} values in {column}:")
    results[i].show()