在spark中为dataframe中的特定列应用逻辑

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, expr

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例DataFrame
data = [("Alice", 25), ("Bob", 30), ("Charlie", 35)]
df = spark.createDataFrame(data, ["name", "age"])

# 使用withColumn方法为age列应用逻辑
df = df.withColumn("age_plus_10", col("age") + 10)

# 显示结果
df.show()

+-------+---+------------+
|   name|age|age_plus_10 |
+-------+---+------------+
|  Alice| 25|          35|
|    Bob| 30|          40|
|Charlie| 35|          45|
+-------+---+------------+

# 使用select方法和表达式选择和转换特定列
df = df.select(col("name"), col("age"), (col("age") + 10).alias("age_plus_10"))

# 显示结果
df.show()