修改/更新pyspark列值

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, lit

# 初始化Spark会话
spark = SparkSession.builder.appName("example").getOrCreate()

# 创建一个示例DataFrame
data = [("Alice", 1), ("Bob", 2), ("Cathy", 3)]
columns = ["name", "age"]
df = spark.createDataFrame(data, columns)

# 使用withColumn更新列值
df_updated = df.withColumn("age", col("age") + 1)  # 将年龄增加1

df_updated.show()

from pyspark.sql.functions import col

# 使用select更新列值
df_updated = df.select(col("name"), col("age") + 1)

df_updated.show()

# 使用rdd.map更新列值
df_rdd = df.rdd.map(lambda row: (row.name, row.age + 1))
df_updated = spark.createDataFrame(df_rdd, schema=columns)

df_updated.show()

from pyspark.sql.functions import when

# 使用withColumn和when进行条件更新
df_updated = df.withColumn("age", when(col("age") > 2, col("age") - 1).otherwise(col("age")))

df_updated.show()