Pyspark计数空值特定列值

from pyspark.sql import SparkSession
from pyspark.sql.functions import col, sum

# 创建 SparkSession
spark = SparkSession.builder.appName("CountNullValues").getOrCreate()

# 创建一个示例 DataFrame
data = [(1, "Alice", None), (2, "Bob", "Engineer"), (3, None, "Doctor"), (4, "David", None)]
columns = ["id", "name", "occupation"]
df = spark.createDataFrame(data, columns)

# 计算特定列（例如 "occupation"）中的空值数量
null_count = df.filter(col("occupation").isNull()).count()
print(f"Null values in 'occupation' column: {null_count}")

# 或者使用 sum 和 isNull 函数来计算空值数量
null_count_sum = df.select(sum(col("occupation").isNull().cast("int"))).collect()[0][0]
print(f"Null values in 'occupation' column using sum: {null_count_sum}")