在一列中获取spark dataframe的所有非空列

# 导入必要的库
from pyspark.sql import SparkSession

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例DataFrame
data = [("Alice", 25, "Engineer"),
        ("Bob", None, "Developer"),
        (None, 30, "Manager")]

df = spark.createDataFrame(data, ["Name", "Age", "Role"])

# 获取DataFrame的所有列名
all_columns = df.columns

# 过滤掉包含空值的列
non_null_columns = [col for col in all_columns if df.dropna(subset=[col]).count() > 0]

# 打印非空列的列表
print(non_null_columns)

['Name', 'Age', 'Role']