pyspark使用sql查询并执行group by优化

from pyspark.sql.functions import broadcast

df1 = ...
df2 = ...

result = df1.join(broadcast(df2), on="join_key")

df.cache()

df.repartition("column_name")

from pyspark.sql.functions import sum, avg

result = df.groupBy("column_name").agg(sum("column1"), avg("column2"))

from pys茂k.sql.window import Window
from pyspark.sql.functions import row_number

windowSpec = Window.partitionBy("column_name").orderBy("column2")
result = df.withColumn("row_number", row_number().over(windowSpec))

query = """
SELECT column1, SUM(column2)
FROM table1
JOIN table2 ON table1.join_key = table2.join_key
GROUP BY column1
"""
result = spark.sql(query)

result.explain()

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("Optimized GroupBy").config("spark.sql.shuffle.partitions", "200").getOrCreate()