合并spark dataframe中的重复列

df1 = df1.select([col(c).alias(c + "_df1") for c in df1.columns])
df2 = df2.select([col(c).alias(c + "_df2") for c in df2.columns])

merged_df = df1.join(df2, df1.common_column_df1 == df2.common_column_df2, "inner")

merged_df = merged_df.select([col(c).alias(c.replace("_df1", "")).alias(c.replace("_df2", "")) for c in merged_df.columns])

from pyspark.sql import SparkSession
from pyspark.sql.functions import col

# 创建SparkSession
spark = SparkSession.builder.getOrCreate()

# 创建示例DataFrame
data1 = [("Alice", 25, "New York"), ("Bob", 30, "San Francisco")]
df1 = spark.createDataFrame(data1, ["name", "age", "city"])

data2 = [("Alice", "Engineer"), ("Bob", "Doctor")]
df2 = spark.createDataFrame(data2, ["name", "profession"])

# 为重复列创建别名并合并DataFrame
df1 = df1.select([col(c).alias(c + "_df1") for c in df1.columns])
df2 = df2.select([col(c).alias(c + "_df2") for c in df2.columns])

merged_df = df1.join(df2, df1.name_df1 == df2.name_df2, "inner")
merged_df = merged_df.select([col(c).alias(c.replace("_df1", "")).alias(c.replace("_df2", "")) for c in merged_df.columns])

# 打印合并后的DataFrame
merged_df.show()