从scala中的其他两个数据集的特定列创建新的数据集

import org.apache.spark.sql.SparkSession

val spark = SparkSession.builder()
  .appName("Create New Dataset from Specific Columns in Scala")
  .getOrCreate()

// 创建两个DataFrame
val df1 = spark.read.format("csv").option("header", "true").load("path/to/dataset1.csv")
val df2 = spark.read.format("csv").option("header", "true").load("path/to/dataset2.csv")

// 选择特定列创建新的DataFrame
val newDF = df1.select("column1", "column2").join(df2.select("column3", "column4"), df1("column1") === df2("column3"))

// 显示新的DataFrame
newDF.show()

import org.apache.spark.sql.SparkSession

val spark = SparkSession.builder()
  .appName("Create New Dataset from Specific Columns in Scala")
  .getOrCreate()

// 创建两个Dataset
val ds1 = spark.read.format("csv").option("header", "true").load("path/to/dataset1.csv").as[MyClass1]
val ds2 = spark.read.format("csv").option("header", "true").load("path/to/dataset2.csv").as[MyClass2]

// 选择特定列创建新的Dataset
val newDS = ds1.select(ds1("column1"), ds1("column2")).join(ds2.select(ds2("column3"), ds2("column4")), ds1("column1") === ds2("column3"))

// 显示新的Dataset
newDS.show()